blob: a7b43780ebf07202a1b4d3fd33d9f48ed4bdbb47 [file] [log] [blame]
Thomas Gleixnerd2912cb2019-06-04 10:11:33 +02001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/drivers/cpufreq/cpufreq.c
4 *
5 * Copyright (C) 2001 Russell King
6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
Viresh Kumarbb176f72013-06-19 14:19:33 +05307 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 *
Ashok Rajc32b6b82005-10-30 14:59:54 -08009 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
Dave Jones32ee8c32006-02-28 00:43:23 -050010 * Added handling for CPU hotplug
Dave Jones8ff69732006-03-05 03:37:23 -050011 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12 * Fix handling for CPU hotplug -- affected CPUs
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 */
14
Viresh Kumardb701152012-10-23 01:29:03 +020015#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
Viresh Kumar5ff0a262013-08-06 22:53:03 +053017#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/cpufreq.h>
Connor O'Brien8bea10e2018-01-31 18:11:57 -080019#include <linux/cpufreq_times.h>
Amit Kucheria5c238a82019-01-30 10:52:01 +053020#include <linux/cpu_cooling.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/device.h>
Viresh Kumar5ff0a262013-08-06 22:53:03 +053023#include <linux/init.h>
24#include <linux/kernel_stat.h>
25#include <linux/module.h>
akpm@osdl.org3fc54d32006-01-13 15:54:22 -080026#include <linux/mutex.h>
Viresh Kumar67d874c2019-07-08 16:27:52 +053027#include <linux/pm_qos.h>
Viresh Kumar5ff0a262013-08-06 22:53:03 +053028#include <linux/slab.h>
Viresh Kumar2f0aea92014-03-04 11:00:26 +080029#include <linux/suspend.h>
Doug Anderson90de2a42014-12-23 22:09:48 -080030#include <linux/syscore_ops.h>
Viresh Kumar5ff0a262013-08-06 22:53:03 +053031#include <linux/tick.h>
Thomas Renninger6f4f2722010-04-20 13:17:36 +020032#include <trace/events/power.h>
Todd Kjosefab33402020-11-23 11:08:48 -080033#include <trace/hooks/cpufreq.h>
Thomas Renninger6f4f2722010-04-20 13:17:36 +020034
Viresh Kumarb4f06762015-01-27 14:06:08 +053035static LIST_HEAD(cpufreq_policy_list);
Viresh Kumarf9637352015-05-12 12:20:11 +053036
Viresh Kumarf9637352015-05-12 12:20:11 +053037/* Macros to iterate over CPU policies */
Eric Biggersfd7dc7e2016-02-21 12:53:12 -060038#define for_each_suitable_policy(__policy, __active) \
39 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
40 if ((__active) == !policy_is_inactive(__policy))
Viresh Kumarf9637352015-05-12 12:20:11 +053041
42#define for_each_active_policy(__policy) \
43 for_each_suitable_policy(__policy, true)
44#define for_each_inactive_policy(__policy) \
45 for_each_suitable_policy(__policy, false)
46
47#define for_each_policy(__policy) \
Viresh Kumarb4f06762015-01-27 14:06:08 +053048 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
49
Viresh Kumarf7b27062015-01-27 14:06:09 +053050/* Iterate over governors */
51static LIST_HEAD(cpufreq_governor_list);
52#define for_each_governor(__governor) \
53 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
54
Quentin Perret8412b452020-06-29 13:55:00 +053055static char default_governor[CPUFREQ_NAME_LEN];
56
Lee Jonesa9909c212020-07-15 09:26:23 +010057/*
Dave Jonescd878472006-08-11 17:59:28 -040058 * The "cpufreq driver" - the arch- or hardware-dependent low
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 * level driver of CPUFreq support, and its spinlock. This lock
60 * also protects the cpufreq_cpu_data array.
61 */
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +020062static struct cpufreq_driver *cpufreq_driver;
Mike Travis7a6aedf2008-03-25 15:06:53 -070063static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
Viresh Kumarbb176f72013-06-19 14:19:33 +053064static DEFINE_RWLOCK(cpufreq_driver_lock);
Viresh Kumarbb176f72013-06-19 14:19:33 +053065
Ionela Voinescu874f6352020-09-01 21:55:47 +010066static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
67bool cpufreq_supports_freq_invariance(void)
68{
69 return static_branch_likely(&cpufreq_freq_invariance);
70}
71
Viresh Kumar2f0aea92014-03-04 11:00:26 +080072/* Flag to suspend/resume CPUFreq governors */
73static bool cpufreq_suspended;
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +053075static inline bool has_target(void)
76{
77 return cpufreq_driver->target_index || cpufreq_driver->target;
78}
79
Linus Torvalds1da177e2005-04-16 15:20:36 -070080/* internal prototypes */
Viresh Kumard92d50a2015-01-02 12:34:29 +053081static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +020082static int cpufreq_init_governor(struct cpufreq_policy *policy);
83static void cpufreq_exit_governor(struct cpufreq_policy *policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +020084static void cpufreq_governor_limits(struct cpufreq_policy *policy);
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +010085static int cpufreq_set_policy(struct cpufreq_policy *policy,
86 struct cpufreq_governor *new_gov,
87 unsigned int new_pol);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +020088
Lee Jonesa9909c212020-07-15 09:26:23 +010089/*
Dave Jones32ee8c32006-02-28 00:43:23 -050090 * Two notifier lists: the "policy" list is involved in the
91 * validation process for a new CPU frequency policy; the
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 * "transition" list for kernel code that needs to handle
93 * changes to devices when the CPU clock speed changes.
94 * The mutex locks both lists.
95 */
Alan Sterne041c682006-03-27 01:16:30 -080096static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
Sebastian Andrzej Siewiorcc85de32018-05-25 12:19:58 +020097SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070098
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -040099static int off __read_mostly;
Viresh Kumarda584452012-10-26 00:51:32 +0200100static int cpufreq_disabled(void)
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -0400101{
102 return off;
103}
104void disable_cpufreq(void)
105{
106 off = 1;
107}
Dave Jones29464f22009-01-18 01:37:11 -0500108static DEFINE_MUTEX(cpufreq_governor_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
Viresh Kumar4d5dcc42013-03-27 15:58:58 +0000110bool have_governor_per_policy(void)
111{
Viresh Kumar0b981e72013-10-02 14:13:18 +0530112 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
Viresh Kumar4d5dcc42013-03-27 15:58:58 +0000113}
Viresh Kumar3f869d62013-05-16 05:09:56 +0000114EXPORT_SYMBOL_GPL(have_governor_per_policy);
Viresh Kumar4d5dcc42013-03-27 15:58:58 +0000115
Yangtao Li183edb202020-02-03 15:45:17 +0000116static struct kobject *cpufreq_global_kobject;
117
Viresh Kumar944e9a02013-05-16 05:09:57 +0000118struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
119{
120 if (have_governor_per_policy())
121 return &policy->kobj;
122 else
123 return cpufreq_global_kobject;
124}
125EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
126
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000127static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
128{
Frederic Weisbecker57208212019-11-21 03:44:28 +0100129 struct kernel_cpustat kcpustat;
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000130 u64 cur_wall_time;
Frederic Weisbecker57208212019-11-21 03:44:28 +0100131 u64 idle_time;
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000132 u64 busy_time;
133
Frederic Weisbecker7fb13272017-01-31 04:09:19 +0100134 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000135
Frederic Weisbecker57208212019-11-21 03:44:28 +0100136 kcpustat_cpu_fetch(&kcpustat, cpu);
137
138 busy_time = kcpustat.cpustat[CPUTIME_USER];
139 busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
140 busy_time += kcpustat.cpustat[CPUTIME_IRQ];
141 busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
142 busy_time += kcpustat.cpustat[CPUTIME_STEAL];
143 busy_time += kcpustat.cpustat[CPUTIME_NICE];
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000144
145 idle_time = cur_wall_time - busy_time;
146 if (wall)
Frederic Weisbecker7fb13272017-01-31 04:09:19 +0100147 *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000148
Frederic Weisbecker7fb13272017-01-31 04:09:19 +0100149 return div_u64(idle_time, NSEC_PER_USEC);
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000150}
151
152u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
153{
154 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
155
156 if (idle_time == -1ULL)
157 return get_cpu_idle_time_jiffy(cpu, wall);
158 else if (!io_busy)
159 idle_time += get_cpu_iowait_time_us(cpu, wall);
160
161 return idle_time;
162}
163EXPORT_SYMBOL_GPL(get_cpu_idle_time);
164
Viresh Kumar70e9e772013-10-03 20:29:07 +0530165/*
166 * This is a generic cpufreq init() routine which can be used by cpufreq
167 * drivers of SMP systems. It will do following:
168 * - validate & show freq table passed
169 * - set policies transition latency
170 * - policy->cpus with all possible CPUs
171 */
Viresh Kumarc4dcc8a2019-07-16 09:36:08 +0530172void cpufreq_generic_init(struct cpufreq_policy *policy,
Viresh Kumar70e9e772013-10-03 20:29:07 +0530173 struct cpufreq_frequency_table *table,
174 unsigned int transition_latency)
175{
Viresh Kumar92c99d12018-02-26 10:38:45 +0530176 policy->freq_table = table;
Viresh Kumar70e9e772013-10-03 20:29:07 +0530177 policy->cpuinfo.transition_latency = transition_latency;
178
179 /*
Shailendra Verma58405af2015-05-22 22:48:22 +0530180 * The driver only supports the SMP configuration where all processors
Viresh Kumar70e9e772013-10-03 20:29:07 +0530181 * share the clock and voltage and clock.
182 */
183 cpumask_setall(policy->cpus);
Viresh Kumar70e9e772013-10-03 20:29:07 +0530184}
185EXPORT_SYMBOL_GPL(cpufreq_generic_init);
186
Rafael J. Wysocki1f0bd442015-09-16 02:17:49 +0200187struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
Viresh Kumar652ed952014-01-09 20:38:43 +0530188{
189 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
190
Viresh Kumar988bed02015-05-08 11:53:45 +0530191 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
192}
Rafael J. Wysocki1f0bd442015-09-16 02:17:49 +0200193EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
Viresh Kumar988bed02015-05-08 11:53:45 +0530194
195unsigned int cpufreq_generic_get(unsigned int cpu)
196{
197 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
198
Viresh Kumar652ed952014-01-09 20:38:43 +0530199 if (!policy || IS_ERR(policy->clk)) {
Joe Perchese837f9b2014-03-11 10:03:00 -0700200 pr_err("%s: No %s associated to cpu: %d\n",
201 __func__, policy ? "clk" : "policy", cpu);
Viresh Kumar652ed952014-01-09 20:38:43 +0530202 return 0;
203 }
204
205 return clk_get_rate(policy->clk) / 1000;
206}
207EXPORT_SYMBOL_GPL(cpufreq_generic_get);
208
Viresh Kumar50e9c852015-02-19 17:02:03 +0530209/**
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100210 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
211 * @cpu: CPU to find the policy for.
Viresh Kumar50e9c852015-02-19 17:02:03 +0530212 *
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100213 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
214 * the kobject reference counter of that policy. Return a valid policy on
215 * success or NULL on failure.
Viresh Kumar50e9c852015-02-19 17:02:03 +0530216 *
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100217 * The policy returned by this function has to be released with the help of
218 * cpufreq_cpu_put() to balance its kobject reference counter properly.
Viresh Kumar50e9c852015-02-19 17:02:03 +0530219 */
Viresh Kumar6eed9402013-08-06 22:53:11 +0530220struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
Viresh Kumar6eed9402013-08-06 22:53:11 +0530222 struct cpufreq_policy *policy = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 unsigned long flags;
224
Viresh Kumar1b947c902015-02-19 17:02:05 +0530225 if (WARN_ON(cpu >= nr_cpu_ids))
Viresh Kumar6eed9402013-08-06 22:53:11 +0530226 return NULL;
227
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 /* get the cpufreq driver */
Nathan Zimmer0d1857a2013-02-22 16:24:34 +0000229 read_lock_irqsave(&cpufreq_driver_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
Viresh Kumar6eed9402013-08-06 22:53:11 +0530231 if (cpufreq_driver) {
232 /* get the CPU */
Viresh Kumar988bed02015-05-08 11:53:45 +0530233 policy = cpufreq_cpu_get_raw(cpu);
Viresh Kumar6eed9402013-08-06 22:53:11 +0530234 if (policy)
235 kobject_get(&policy->kobj);
236 }
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +0200237
Viresh Kumar6eed9402013-08-06 22:53:11 +0530238 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
Viresh Kumar3a3e9e02013-08-06 22:53:05 +0530240 return policy;
Stephen Boyda9144432012-07-20 18:14:38 +0000241}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
243
Viresh Kumar50e9c852015-02-19 17:02:03 +0530244/**
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100245 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
246 * @policy: cpufreq policy returned by cpufreq_cpu_get().
Viresh Kumar50e9c852015-02-19 17:02:03 +0530247 */
Viresh Kumar3a3e9e02013-08-06 22:53:05 +0530248void cpufreq_cpu_put(struct cpufreq_policy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249{
Viresh Kumar6eed9402013-08-06 22:53:11 +0530250 kobject_put(&policy->kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251}
252EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
253
Rafael J. Wysocki540a37582019-03-26 12:16:58 +0100254/**
255 * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
256 * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
257 */
Rafael J. Wysocki9083e492019-03-26 12:19:52 +0100258void cpufreq_cpu_release(struct cpufreq_policy *policy)
Rafael J. Wysocki540a37582019-03-26 12:16:58 +0100259{
260 if (WARN_ON(!policy))
261 return;
262
263 lockdep_assert_held(&policy->rwsem);
264
265 up_write(&policy->rwsem);
266
267 cpufreq_cpu_put(policy);
268}
269
270/**
271 * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
272 * @cpu: CPU to find the policy for.
273 *
274 * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
275 * if the policy returned by it is not NULL, acquire its rwsem for writing.
276 * Return the policy if it is active or release it and return NULL otherwise.
277 *
278 * The policy returned by this function has to be released with the help of
279 * cpufreq_cpu_release() in order to release its rwsem and balance its usage
280 * counter properly.
281 */
Rafael J. Wysocki9083e492019-03-26 12:19:52 +0100282struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
Rafael J. Wysocki540a37582019-03-26 12:16:58 +0100283{
284 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
285
286 if (!policy)
287 return NULL;
288
289 down_write(&policy->rwsem);
290
291 if (policy_is_inactive(policy)) {
292 cpufreq_cpu_release(policy);
293 return NULL;
294 }
295
296 return policy;
297}
298
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299/*********************************************************************
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
301 *********************************************************************/
302
Lee Jonesa9909c212020-07-15 09:26:23 +0100303/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 * adjust_jiffies - adjust the system "loops_per_jiffy"
305 *
306 * This function alters the system "loops_per_jiffy" for the clock
307 * speed change. Note that loops_per_jiffy cannot be updated on SMP
Dave Jones32ee8c32006-02-28 00:43:23 -0500308 * systems as each CPU might be scaled differently. So, use the arch
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 * per-CPU loops_per_jiffy value wherever possible.
310 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800311static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312{
Viresh Kumar39c132e2015-01-02 12:34:34 +0530313#ifndef CONFIG_SMP
314 static unsigned long l_p_j_ref;
315 static unsigned int l_p_j_ref_freq;
316
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 if (ci->flags & CPUFREQ_CONST_LOOPS)
318 return;
319
320 if (!l_p_j_ref_freq) {
321 l_p_j_ref = loops_per_jiffy;
322 l_p_j_ref_freq = ci->old;
Joe Perchese837f9b2014-03-11 10:03:00 -0700323 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
324 l_p_j_ref, l_p_j_ref_freq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 }
Viresh Kumar0b443ea2014-03-19 11:24:58 +0530326 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530327 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
328 ci->new);
Joe Perchese837f9b2014-03-11 10:03:00 -0700329 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
330 loops_per_jiffy, ci->new);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332#endif
Viresh Kumar39c132e2015-01-02 12:34:34 +0530333}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334
Viresh Kumar20b53242018-05-10 15:00:29 +0530335/**
336 * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies.
337 * @policy: cpufreq policy to enable fast frequency switching for.
338 * @freqs: contain details of the frequency update.
339 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
340 *
341 * This function calls the transition notifiers and the "adjust_jiffies"
342 * function. It is called twice on all CPU frequency changes that have
343 * external effects.
344 */
345static void cpufreq_notify_transition(struct cpufreq_policy *policy,
346 struct cpufreq_freqs *freqs,
347 unsigned int state)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348{
Viresh Kumardf240142019-04-29 15:03:58 +0530349 int cpu;
350
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 BUG_ON(irqs_disabled());
352
Dirk Brandewied5aaffa2013-01-17 16:22:21 +0000353 if (cpufreq_disabled())
354 return;
355
Viresh Kumardf240142019-04-29 15:03:58 +0530356 freqs->policy = policy;
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +0200357 freqs->flags = cpufreq_driver->flags;
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +0200358 pr_debug("notification %u of frequency transition to %u kHz\n",
Joe Perchese837f9b2014-03-11 10:03:00 -0700359 state, freqs->new);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 switch (state) {
362 case CPUFREQ_PRECHANGE:
Viresh Kumar20b53242018-05-10 15:00:29 +0530363 /*
364 * Detect if the driver reported a value as "old frequency"
Dave Jonese4472cb2006-01-31 15:53:55 -0800365 * which is not equal to what the cpufreq core thinks is
366 * "old frequency".
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 */
Viresh Kumar98015222019-06-28 10:46:55 +0530368 if (policy->cur && policy->cur != freqs->old) {
369 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
370 freqs->old, policy->cur);
371 freqs->old = policy->cur;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 }
Viresh Kumar20b53242018-05-10 15:00:29 +0530373
Viresh Kumardf240142019-04-29 15:03:58 +0530374 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
375 CPUFREQ_PRECHANGE, freqs);
Viresh Kumar20b53242018-05-10 15:00:29 +0530376
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
378 break;
Dave Jonese4472cb2006-01-31 15:53:55 -0800379
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 case CPUFREQ_POSTCHANGE:
381 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
Viresh Kumar20b53242018-05-10 15:00:29 +0530382 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
383 cpumask_pr_args(policy->cpus));
Viresh Kumarbb176f72013-06-19 14:19:33 +0530384
Viresh Kumardf240142019-04-29 15:03:58 +0530385 for_each_cpu(cpu, policy->cpus)
386 trace_cpu_frequency(freqs->new, cpu);
387
388 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
389 CPUFREQ_POSTCHANGE, freqs);
Viresh Kumar20b53242018-05-10 15:00:29 +0530390
391 cpufreq_stats_record_transition(policy, freqs->new);
Connor O'Brien0d084d72019-02-08 12:30:41 -0800392 cpufreq_times_record_transition(policy, freqs->new);
Viresh Kumar20b53242018-05-10 15:00:29 +0530393 policy->cur = freqs->new;
Pavankumar Kondeti2d450012021-03-05 04:31:50 +0530394 trace_android_rvh_cpufreq_transition(policy);
Viresh Kumar20b53242018-05-10 15:00:29 +0530395 }
Viresh Kumarb43a7ff2013-03-24 11:56:43 +0530396}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397
Viresh Kumarf7ba3b42013-12-02 11:04:12 +0530398/* Do post notifications when there are chances that transition has failed */
Viresh Kumar236a9802014-03-24 13:35:46 +0530399static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
Viresh Kumarf7ba3b42013-12-02 11:04:12 +0530400 struct cpufreq_freqs *freqs, int transition_failed)
401{
402 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
403 if (!transition_failed)
404 return;
405
406 swap(freqs->old, freqs->new);
407 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
408 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
409}
Viresh Kumarf7ba3b42013-12-02 11:04:12 +0530410
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530411void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
412 struct cpufreq_freqs *freqs)
413{
Srivatsa S. Bhatca654dc2014-05-05 12:52:39 +0530414
415 /*
416 * Catch double invocations of _begin() which lead to self-deadlock.
417 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
418 * doesn't invoke _begin() on their behalf, and hence the chances of
419 * double invocations are very low. Moreover, there are scenarios
420 * where these checks can emit false-positive warnings in these
421 * drivers; so we avoid that by skipping them altogether.
422 */
423 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
424 && current == policy->transition_task);
425
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530426wait:
427 wait_event(policy->transition_wait, !policy->transition_ongoing);
428
429 spin_lock(&policy->transition_lock);
430
431 if (unlikely(policy->transition_ongoing)) {
432 spin_unlock(&policy->transition_lock);
433 goto wait;
434 }
435
436 policy->transition_ongoing = true;
Srivatsa S. Bhatca654dc2014-05-05 12:52:39 +0530437 policy->transition_task = current;
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530438
439 spin_unlock(&policy->transition_lock);
440
441 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
442}
443EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
444
445void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
446 struct cpufreq_freqs *freqs, int transition_failed)
447{
Igor Stoppa0e7ea2f2018-09-07 19:09:55 +0300448 if (WARN_ON(!policy->transition_ongoing))
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530449 return;
450
451 cpufreq_notify_post_transition(policy, freqs, transition_failed);
452
Ionela Voinescu1a0419b2020-09-01 21:55:46 +0100453 arch_set_freq_scale(policy->related_cpus,
454 policy->cur,
455 policy->cpuinfo.max_freq);
456
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530457 policy->transition_ongoing = false;
Srivatsa S. Bhatca654dc2014-05-05 12:52:39 +0530458 policy->transition_task = NULL;
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530459
460 wake_up(&policy->transition_wait);
461}
462EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
463
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +0200464/*
465 * Fast frequency switching status count. Positive means "enabled", negative
466 * means "disabled" and 0 means "not decided yet".
467 */
468static int cpufreq_fast_switch_count;
469static DEFINE_MUTEX(cpufreq_fast_switch_lock);
470
471static void cpufreq_list_transition_notifiers(void)
472{
473 struct notifier_block *nb;
474
475 pr_info("Registered transition notifiers:\n");
476
477 mutex_lock(&cpufreq_transition_notifier_list.mutex);
478
479 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
Sakari Ailusd75f7732019-03-25 21:32:28 +0200480 pr_info("%pS\n", nb->notifier_call);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +0200481
482 mutex_unlock(&cpufreq_transition_notifier_list.mutex);
483}
484
485/**
486 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
487 * @policy: cpufreq policy to enable fast frequency switching for.
488 *
489 * Try to enable fast frequency switching for @policy.
490 *
491 * The attempt will fail if there is at least one transition notifier registered
492 * at this point, as fast frequency switching is quite fundamentally at odds
493 * with transition notifiers. Thus if successful, it will make registration of
494 * transition notifiers fail going forward.
495 */
496void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
497{
498 lockdep_assert_held(&policy->rwsem);
499
500 if (!policy->fast_switch_possible)
501 return;
502
503 mutex_lock(&cpufreq_fast_switch_lock);
504 if (cpufreq_fast_switch_count >= 0) {
505 cpufreq_fast_switch_count++;
506 policy->fast_switch_enabled = true;
507 } else {
508 pr_warn("CPU%u: Fast frequency switching not enabled\n",
509 policy->cpu);
510 cpufreq_list_transition_notifiers();
511 }
512 mutex_unlock(&cpufreq_fast_switch_lock);
513}
514EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
515
Rafael J. Wysocki6c9d9c82016-04-07 23:38:46 +0200516/**
517 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
518 * @policy: cpufreq policy to disable fast frequency switching for.
519 */
520void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +0200521{
522 mutex_lock(&cpufreq_fast_switch_lock);
523 if (policy->fast_switch_enabled) {
524 policy->fast_switch_enabled = false;
525 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
526 cpufreq_fast_switch_count--;
527 }
528 mutex_unlock(&cpufreq_fast_switch_lock);
529}
Rafael J. Wysocki6c9d9c82016-04-07 23:38:46 +0200530EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531
Steve Mucklee3c06232016-07-13 13:25:25 -0700532/**
533 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
534 * one.
Lee Jonesa9909c212020-07-15 09:26:23 +0100535 * @policy: associated policy to interrogate
Steve Mucklee3c06232016-07-13 13:25:25 -0700536 * @target_freq: target frequency to resolve.
537 *
538 * The target to driver frequency mapping is cached in the policy.
539 *
540 * Return: Lowest driver-supported frequency greater than or equal to the
541 * given target_freq, subject to policy (min/max) and driver limitations.
542 */
543unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
544 unsigned int target_freq)
545{
546 target_freq = clamp_val(target_freq, policy->min, policy->max);
547 policy->cached_target_freq = target_freq;
Viresh Kumarabe8bd02016-07-21 14:39:26 -0700548
549 if (cpufreq_driver->target_index) {
Viresh Kumar292072c2020-07-30 08:59:40 +0530550 unsigned int idx;
Viresh Kumarabe8bd02016-07-21 14:39:26 -0700551
552 idx = cpufreq_frequency_table_target(policy, target_freq,
553 CPUFREQ_RELATION_L);
554 policy->cached_resolved_idx = idx;
555 return policy->freq_table[idx].frequency;
556 }
557
Steve Mucklee3c06232016-07-13 13:25:25 -0700558 if (cpufreq_driver->resolve_freq)
559 return cpufreq_driver->resolve_freq(policy, target_freq);
Viresh Kumarabe8bd02016-07-21 14:39:26 -0700560
561 return target_freq;
Steve Mucklee3c06232016-07-13 13:25:25 -0700562}
Steve Muckleae2c1ca2016-07-21 19:24:38 -0700563EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
Steve Mucklee3c06232016-07-13 13:25:25 -0700564
Viresh Kumaraa7519a2017-07-19 15:42:42 +0530565unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
566{
567 unsigned int latency;
568
569 if (policy->transition_delay_us)
570 return policy->transition_delay_us;
571
572 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
Viresh Kumare948bc82017-08-17 09:12:27 +0530573 if (latency) {
574 /*
575 * For platforms that can change the frequency very fast (< 10
576 * us), the above formula gives a decent transition delay. But
577 * for platforms where transition_latency is in milliseconds, it
578 * ends up giving unrealistic values.
579 *
580 * Cap the default transition delay to 10 ms, which seems to be
581 * a reasonable amount of time after which we should reevaluate
582 * the frequency.
583 */
584 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
585 }
Viresh Kumaraa7519a2017-07-19 15:42:42 +0530586
587 return LATENCY_MULTIPLIER;
588}
589EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
590
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591/*********************************************************************
592 * SYSFS INTERFACE *
593 *********************************************************************/
Rashika Kheria8a5c74a2014-02-26 22:12:42 +0530594static ssize_t show_boost(struct kobject *kobj,
Viresh Kumar625c85a2019-01-25 12:53:07 +0530595 struct kobj_attribute *attr, char *buf)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100596{
597 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
598}
599
Viresh Kumar625c85a2019-01-25 12:53:07 +0530600static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
601 const char *buf, size_t count)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100602{
603 int ret, enable;
604
605 ret = sscanf(buf, "%d", &enable);
606 if (ret != 1 || enable < 0 || enable > 1)
607 return -EINVAL;
608
609 if (cpufreq_boost_trigger_state(enable)) {
Joe Perchese837f9b2014-03-11 10:03:00 -0700610 pr_err("%s: Cannot %s BOOST!\n",
611 __func__, enable ? "enable" : "disable");
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100612 return -EINVAL;
613 }
614
Joe Perchese837f9b2014-03-11 10:03:00 -0700615 pr_debug("%s: cpufreq BOOST %s\n",
616 __func__, enable ? "enabled" : "disabled");
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100617
618 return count;
619}
620define_one_global_rw(boost);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621
Viresh Kumar42f91fa2015-01-02 12:34:26 +0530622static struct cpufreq_governor *find_governor(const char *str_governor)
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -0700623{
624 struct cpufreq_governor *t;
625
Viresh Kumarf7b27062015-01-27 14:06:09 +0530626 for_each_governor(t)
Rasmus Villemoes7c4f4532014-09-29 15:50:11 +0200627 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -0700628 return t;
629
630 return NULL;
631}
632
Viresh Kumar8cc46ae2020-06-29 13:54:58 +0530633static struct cpufreq_governor *get_governor(const char *str_governor)
634{
635 struct cpufreq_governor *t;
636
637 mutex_lock(&cpufreq_governor_mutex);
638 t = find_governor(str_governor);
639 if (!t)
640 goto unlock;
641
642 if (!try_module_get(t->owner))
643 t = NULL;
644
645unlock:
646 mutex_unlock(&cpufreq_governor_mutex);
647
648 return t;
649}
650
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100651static unsigned int cpufreq_parse_policy(char *str_governor)
Yue Huab05d972019-04-29 15:24:18 +0800652{
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100653 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN))
654 return CPUFREQ_POLICY_PERFORMANCE;
655
656 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN))
657 return CPUFREQ_POLICY_POWERSAVE;
658
659 return CPUFREQ_POLICY_UNKNOWN;
Yue Huab05d972019-04-29 15:24:18 +0800660}
661
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662/**
Viresh Kumar5ddc6d42019-06-20 08:35:48 +0530663 * cpufreq_parse_governor - parse a governor string only for has_target()
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100664 * @str_governor: Governor name.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 */
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100666static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667{
Yue Huab05d972019-04-29 15:24:18 +0800668 struct cpufreq_governor *t;
Rafael J. Wysocki045149e2017-11-23 01:23:16 +0100669
Viresh Kumar8cc46ae2020-06-29 13:54:58 +0530670 t = get_governor(str_governor);
671 if (t)
672 return t;
Yue Huab05d972019-04-29 15:24:18 +0800673
Viresh Kumar8cc46ae2020-06-29 13:54:58 +0530674 if (request_module("cpufreq_%s", str_governor))
675 return NULL;
Yue Huab05d972019-04-29 15:24:18 +0800676
Viresh Kumar8cc46ae2020-06-29 13:54:58 +0530677 return get_governor(str_governor);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679
Lee Jonesa9909c212020-07-15 09:26:23 +0100680/*
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530681 * cpufreq_per_cpu_attr_read() / show_##file_name() -
682 * print out cpufreq information
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 *
684 * Write out information from cpufreq_driver->policy[cpu]; object must be
685 * "unsigned int".
686 */
687
Dave Jones32ee8c32006-02-28 00:43:23 -0500688#define show_one(file_name, object) \
689static ssize_t show_##file_name \
Dave Jones905d77c2008-03-05 14:28:32 -0500690(struct cpufreq_policy *policy, char *buf) \
Dave Jones32ee8c32006-02-28 00:43:23 -0500691{ \
Dave Jones29464f22009-01-18 01:37:11 -0500692 return sprintf(buf, "%u\n", policy->object); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693}
694
Todd Kjosefab33402020-11-23 11:08:48 -0800695static ssize_t show_cpuinfo_max_freq(struct cpufreq_policy *policy, char *buf)
696{
697 unsigned int max_freq = policy->cpuinfo.max_freq;
698
699 trace_android_vh_show_max_freq(policy, &max_freq);
Shaleen Agrawalf3e3a452021-07-22 17:09:23 -0700700 trace_android_rvh_show_max_freq(policy, &max_freq);
Todd Kjosefab33402020-11-23 11:08:48 -0800701 return sprintf(buf, "%u\n", max_freq);
702}
703
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704show_one(cpuinfo_min_freq, cpuinfo.min_freq);
Thomas Renningered129782009-02-04 01:17:41 +0100705show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706show_one(scaling_min_freq, min);
707show_one(scaling_max_freq, max);
Dirk Brandewiec034b022014-10-13 08:37:40 -0700708
Len Brownf8475ce2017-06-23 22:11:52 -0700709__weak unsigned int arch_freq_get_on_cpu(int cpu)
710{
711 return 0;
712}
713
Viresh Kumar09347b22015-01-02 12:34:24 +0530714static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
Dirk Brandewiec034b022014-10-13 08:37:40 -0700715{
716 ssize_t ret;
Len Brownf8475ce2017-06-23 22:11:52 -0700717 unsigned int freq;
Dirk Brandewiec034b022014-10-13 08:37:40 -0700718
Len Brownf8475ce2017-06-23 22:11:52 -0700719 freq = arch_freq_get_on_cpu(policy->cpu);
720 if (freq)
721 ret = sprintf(buf, "%u\n", freq);
Viresh Kumar681fe682020-08-27 10:54:15 +0530722 else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
Dirk Brandewiec034b022014-10-13 08:37:40 -0700723 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
724 else
725 ret = sprintf(buf, "%u\n", policy->cur);
726 return ret;
727}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728
Lee Jonesa9909c212020-07-15 09:26:23 +0100729/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
731 */
732#define store_one(file_name, object) \
733static ssize_t store_##file_name \
Dave Jones905d77c2008-03-05 14:28:32 -0500734(struct cpufreq_policy *policy, const char *buf, size_t count) \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735{ \
Viresh Kumar18c49922019-07-05 16:21:24 +0530736 unsigned long val; \
737 int ret; \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 \
Viresh Kumar18c49922019-07-05 16:21:24 +0530739 ret = sscanf(buf, "%lu", &val); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 if (ret != 1) \
741 return -EINVAL; \
742 \
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +0200743 ret = freq_qos_update_request(policy->object##_freq_req, val);\
Viresh Kumar18c49922019-07-05 16:21:24 +0530744 return ret >= 0 ? count : ret; \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745}
746
Dave Jones29464f22009-01-18 01:37:11 -0500747store_one(scaling_min_freq, min);
748store_one(scaling_max_freq, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749
Lee Jonesa9909c212020-07-15 09:26:23 +0100750/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
752 */
Dave Jones905d77c2008-03-05 14:28:32 -0500753static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
754 char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755{
Viresh Kumard92d50a2015-01-02 12:34:29 +0530756 unsigned int cur_freq = __cpufreq_get(policy);
Rafael J. Wysocki9b4f6032017-03-15 00:12:16 +0100757
758 if (cur_freq)
759 return sprintf(buf, "%u\n", cur_freq);
760
761 return sprintf(buf, "<unknown>\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762}
763
Lee Jonesa9909c212020-07-15 09:26:23 +0100764/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765 * show_scaling_governor - show the current policy for the specified CPU
766 */
Dave Jones905d77c2008-03-05 14:28:32 -0500767static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768{
Dave Jones29464f22009-01-18 01:37:11 -0500769 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 return sprintf(buf, "powersave\n");
771 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
772 return sprintf(buf, "performance\n");
773 else if (policy->governor)
viresh kumar4b972f02012-10-23 01:23:43 +0200774 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
Dave Jones29464f22009-01-18 01:37:11 -0500775 policy->governor->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 return -EINVAL;
777}
778
Lee Jonesa9909c212020-07-15 09:26:23 +0100779/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 * store_scaling_governor - store policy for the specified CPU
781 */
Dave Jones905d77c2008-03-05 14:28:32 -0500782static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
783 const char *buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784{
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100785 char str_governor[16];
Srivatsa S. Bhat5136fa52013-09-07 01:24:06 +0530786 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787
Dave Jones29464f22009-01-18 01:37:11 -0500788 ret = sscanf(buf, "%15s", str_governor);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 if (ret != 1)
790 return -EINVAL;
791
Yue Huab05d972019-04-29 15:24:18 +0800792 if (cpufreq_driver->setpolicy) {
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100793 unsigned int new_pol;
794
795 new_pol = cpufreq_parse_policy(str_governor);
796 if (!new_pol)
Yue Huab05d972019-04-29 15:24:18 +0800797 return -EINVAL;
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100798
799 ret = cpufreq_set_policy(policy, NULL, new_pol);
Yue Huab05d972019-04-29 15:24:18 +0800800 } else {
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100801 struct cpufreq_governor *new_gov;
802
803 new_gov = cpufreq_parse_governor(str_governor);
804 if (!new_gov)
Yue Huab05d972019-04-29 15:24:18 +0800805 return -EINVAL;
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +0100806
807 ret = cpufreq_set_policy(policy, new_gov,
808 CPUFREQ_POLICY_UNKNOWN);
809
810 module_put(new_gov->owner);
Yue Huab05d972019-04-29 15:24:18 +0800811 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
Viresh Kumar88dc4382015-08-03 08:36:18 +0530813 return ret ? ret : count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814}
815
Lee Jonesa9909c212020-07-15 09:26:23 +0100816/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 * show_scaling_driver - show the cpufreq driver currently loaded
818 */
Dave Jones905d77c2008-03-05 14:28:32 -0500819static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820{
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +0200821 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822}
823
Lee Jonesa9909c212020-07-15 09:26:23 +0100824/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 * show_scaling_available_governors - show the available CPUfreq governors
826 */
Dave Jones905d77c2008-03-05 14:28:32 -0500827static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
828 char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829{
830 ssize_t i = 0;
831 struct cpufreq_governor *t;
832
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +0530833 if (!has_target()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 i += sprintf(buf, "performance powersave");
835 goto out;
836 }
837
Viresh Kumar8cc46ae2020-06-29 13:54:58 +0530838 mutex_lock(&cpufreq_governor_mutex);
Viresh Kumarf7b27062015-01-27 14:06:09 +0530839 for_each_governor(t) {
Dave Jones29464f22009-01-18 01:37:11 -0500840 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
841 - (CPUFREQ_NAME_LEN + 2)))
Viresh Kumar8cc46ae2020-06-29 13:54:58 +0530842 break;
viresh kumar4b972f02012-10-23 01:23:43 +0200843 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844 }
Viresh Kumar8cc46ae2020-06-29 13:54:58 +0530845 mutex_unlock(&cpufreq_governor_mutex);
Dave Jones7d5e3502006-02-02 17:03:42 -0500846out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 i += sprintf(&buf[i], "\n");
848 return i;
849}
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700850
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800851ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852{
853 ssize_t i = 0;
854 unsigned int cpu;
855
Rusty Russell835481d2009-01-04 05:18:06 -0800856 for_each_cpu(cpu, mask) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 if (i)
858 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
859 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
860 if (i >= (PAGE_SIZE - 5))
Dave Jones29464f22009-01-18 01:37:11 -0500861 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862 }
863 i += sprintf(&buf[i], "\n");
864 return i;
865}
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800866EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867
Lee Jonesa9909c212020-07-15 09:26:23 +0100868/*
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700869 * show_related_cpus - show the CPUs affected by each transition even if
870 * hw coordination is in use
871 */
872static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
873{
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800874 return cpufreq_show_cpus(policy->related_cpus, buf);
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700875}
876
Lee Jonesa9909c212020-07-15 09:26:23 +0100877/*
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700878 * show_affected_cpus - show the CPUs affected by each transition
879 */
880static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
881{
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800882 return cpufreq_show_cpus(policy->cpus, buf);
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700883}
884
Venki Pallipadi9e769882007-10-26 10:18:21 -0700885static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
Dave Jones905d77c2008-03-05 14:28:32 -0500886 const char *buf, size_t count)
Venki Pallipadi9e769882007-10-26 10:18:21 -0700887{
888 unsigned int freq = 0;
889 unsigned int ret;
890
CHIKAMA masaki879000f2008-06-05 22:46:33 -0700891 if (!policy->governor || !policy->governor->store_setspeed)
Venki Pallipadi9e769882007-10-26 10:18:21 -0700892 return -EINVAL;
893
894 ret = sscanf(buf, "%u", &freq);
895 if (ret != 1)
896 return -EINVAL;
897
898 policy->governor->store_setspeed(policy, freq);
899
900 return count;
901}
902
903static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
904{
CHIKAMA masaki879000f2008-06-05 22:46:33 -0700905 if (!policy->governor || !policy->governor->show_setspeed)
Venki Pallipadi9e769882007-10-26 10:18:21 -0700906 return sprintf(buf, "<unsupported>\n");
907
908 return policy->governor->show_setspeed(policy, buf);
909}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910
Lee Jonesa9909c212020-07-15 09:26:23 +0100911/*
viresh kumar8bf1ac722012-10-23 01:23:33 +0200912 * show_bios_limit - show the current cpufreq HW/BIOS limitation
Thomas Renningere2f74f32009-11-19 12:31:01 +0100913 */
914static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
915{
916 unsigned int limit;
917 int ret;
Yue Hub23aa312019-04-16 10:40:27 +0800918 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
919 if (!ret)
920 return sprintf(buf, "%u\n", limit);
Thomas Renningere2f74f32009-11-19 12:31:01 +0100921 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
922}
923
Borislav Petkov6dad2a22010-03-31 21:56:46 +0200924cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
925cpufreq_freq_attr_ro(cpuinfo_min_freq);
926cpufreq_freq_attr_ro(cpuinfo_max_freq);
927cpufreq_freq_attr_ro(cpuinfo_transition_latency);
928cpufreq_freq_attr_ro(scaling_available_governors);
929cpufreq_freq_attr_ro(scaling_driver);
930cpufreq_freq_attr_ro(scaling_cur_freq);
931cpufreq_freq_attr_ro(bios_limit);
932cpufreq_freq_attr_ro(related_cpus);
933cpufreq_freq_attr_ro(affected_cpus);
934cpufreq_freq_attr_rw(scaling_min_freq);
935cpufreq_freq_attr_rw(scaling_max_freq);
936cpufreq_freq_attr_rw(scaling_governor);
937cpufreq_freq_attr_rw(scaling_setspeed);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938
Dave Jones905d77c2008-03-05 14:28:32 -0500939static struct attribute *default_attrs[] = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 &cpuinfo_min_freq.attr,
941 &cpuinfo_max_freq.attr,
Thomas Renningered129782009-02-04 01:17:41 +0100942 &cpuinfo_transition_latency.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 &scaling_min_freq.attr,
944 &scaling_max_freq.attr,
945 &affected_cpus.attr,
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700946 &related_cpus.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947 &scaling_governor.attr,
948 &scaling_driver.attr,
949 &scaling_available_governors.attr,
Venki Pallipadi9e769882007-10-26 10:18:21 -0700950 &scaling_setspeed.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 NULL
952};
953
Dave Jones29464f22009-01-18 01:37:11 -0500954#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
955#define to_attr(a) container_of(a, struct freq_attr, attr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956
Dave Jones29464f22009-01-18 01:37:11 -0500957static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958{
Dave Jones905d77c2008-03-05 14:28:32 -0500959 struct cpufreq_policy *policy = to_policy(kobj);
960 struct freq_attr *fattr = to_attr(attr);
Viresh Kumar1b750e32013-10-02 14:13:09 +0530961 ssize_t ret;
Viresh Kumar6eed9402013-08-06 22:53:11 +0530962
Kai Shene6e8df02019-11-07 05:08:17 +0000963 if (!fattr->show)
964 return -EIO;
965
viresh kumarad7722d2013-10-18 19:10:15 +0530966 down_read(&policy->rwsem);
Rafael J. Wysocki6541aef2016-02-12 23:56:21 +0100967 ret = fattr->show(policy, buf);
viresh kumarad7722d2013-10-18 19:10:15 +0530968 up_read(&policy->rwsem);
Viresh Kumar1b750e32013-10-02 14:13:09 +0530969
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970 return ret;
971}
972
Dave Jones905d77c2008-03-05 14:28:32 -0500973static ssize_t store(struct kobject *kobj, struct attribute *attr,
974 const char *buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975{
Dave Jones905d77c2008-03-05 14:28:32 -0500976 struct cpufreq_policy *policy = to_policy(kobj);
977 struct freq_attr *fattr = to_attr(attr);
Dave Jonesa07530b2008-03-05 14:22:25 -0500978 ssize_t ret = -EINVAL;
Viresh Kumar6eed9402013-08-06 22:53:11 +0530979
Kai Shene6e8df02019-11-07 05:08:17 +0000980 if (!fattr->store)
981 return -EIO;
982
Waiman Long9b3d9bb2018-07-24 14:26:05 -0400983 /*
984 * cpus_read_trylock() is used here to work around a circular lock
985 * dependency problem with respect to the cpufreq_register_driver().
986 */
987 if (!cpus_read_trylock())
988 return -EBUSY;
Srivatsa S. Bhat4f750c92013-09-07 01:23:43 +0530989
Rafael J. Wysocki6541aef2016-02-12 23:56:21 +0100990 if (cpu_online(policy->cpu)) {
991 down_write(&policy->rwsem);
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530992 ret = fattr->store(policy, buf, count);
Rafael J. Wysocki6541aef2016-02-12 23:56:21 +0100993 up_write(&policy->rwsem);
994 }
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530995
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +0200996 cpus_read_unlock();
Srivatsa S. Bhat4f750c92013-09-07 01:23:43 +0530997
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 return ret;
999}
1000
Dave Jones905d77c2008-03-05 14:28:32 -05001001static void cpufreq_sysfs_release(struct kobject *kobj)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002{
Dave Jones905d77c2008-03-05 14:28:32 -05001003 struct cpufreq_policy *policy = to_policy(kobj);
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02001004 pr_debug("last reference is dropped\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 complete(&policy->kobj_unregister);
1006}
1007
Emese Revfy52cf25d2010-01-19 02:58:23 +01001008static const struct sysfs_ops sysfs_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 .show = show,
1010 .store = store,
1011};
1012
1013static struct kobj_type ktype_cpufreq = {
1014 .sysfs_ops = &sysfs_ops,
1015 .default_attrs = default_attrs,
1016 .release = cpufreq_sysfs_release,
1017};
1018
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001019static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
Viresh Kumar87549142015-06-10 02:13:21 +02001020{
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001021 struct device *dev = get_cpu_device(cpu);
1022
Viresh Kumar67d874c2019-07-08 16:27:52 +05301023 if (unlikely(!dev))
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001024 return;
1025
1026 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
1027 return;
1028
Viresh Kumar266198042016-09-12 12:07:05 +05301029 dev_dbg(dev, "%s: Adding symlink\n", __func__);
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001030 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1031 dev_err(dev, "cpufreq symlink creation failed\n");
Viresh Kumar87549142015-06-10 02:13:21 +02001032}
1033
Viresh Kumar266198042016-09-12 12:07:05 +05301034static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
1035 struct device *dev)
Viresh Kumar87549142015-06-10 02:13:21 +02001036{
Viresh Kumar266198042016-09-12 12:07:05 +05301037 dev_dbg(dev, "%s: Removing symlink\n", __func__);
1038 sysfs_remove_link(&dev->kobj, "cpufreq");
Viresh Kumar87549142015-06-10 02:13:21 +02001039}
1040
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001041static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
Dave Jones909a6942009-07-08 18:05:42 -04001042{
1043 struct freq_attr **drv_attr;
Dave Jones909a6942009-07-08 18:05:42 -04001044 int ret = 0;
Dave Jones909a6942009-07-08 18:05:42 -04001045
Dave Jones909a6942009-07-08 18:05:42 -04001046 /* set up files for this cpu device */
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001047 drv_attr = cpufreq_driver->attr;
Viresh Kumarf13f1182015-01-02 12:34:23 +05301048 while (drv_attr && *drv_attr) {
Dave Jones909a6942009-07-08 18:05:42 -04001049 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1050 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001051 return ret;
Dave Jones909a6942009-07-08 18:05:42 -04001052 drv_attr++;
1053 }
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001054 if (cpufreq_driver->get) {
Dave Jones909a6942009-07-08 18:05:42 -04001055 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1056 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001057 return ret;
Dave Jones909a6942009-07-08 18:05:42 -04001058 }
Dirk Brandewiec034b022014-10-13 08:37:40 -07001059
1060 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1061 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001062 return ret;
Dirk Brandewiec034b022014-10-13 08:37:40 -07001063
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001064 if (cpufreq_driver->bios_limit) {
Thomas Renningere2f74f32009-11-19 12:31:01 +01001065 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1066 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001067 return ret;
Thomas Renningere2f74f32009-11-19 12:31:01 +01001068 }
Dave Jones909a6942009-07-08 18:05:42 -04001069
Viresh Kumar266198042016-09-12 12:07:05 +05301070 return 0;
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301071}
1072
Viresh Kumar7f0fa402015-07-08 15:12:16 +05301073static int cpufreq_init_policy(struct cpufreq_policy *policy)
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301074{
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01001075 struct cpufreq_governor *gov = NULL;
1076 unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
Viresh Kumar8cc46ae2020-06-29 13:54:58 +05301077 int ret;
Yue Huab05d972019-04-29 15:24:18 +08001078
1079 if (has_target()) {
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01001080 /* Update policy governor to the one used before hotplug. */
Viresh Kumar8cc46ae2020-06-29 13:54:58 +05301081 gov = get_governor(policy->last_governor);
Yue Huab05d972019-04-29 15:24:18 +08001082 if (gov) {
1083 pr_debug("Restoring governor %s for cpu %d\n",
Quentin Perret8412b452020-06-29 13:55:00 +05301084 gov->name, policy->cpu);
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01001085 } else {
Quentin Perret8412b452020-06-29 13:55:00 +05301086 gov = get_governor(default_governor);
Yue Huab05d972019-04-29 15:24:18 +08001087 }
Quentin Perret8412b452020-06-29 13:55:00 +05301088
1089 if (!gov) {
1090 gov = cpufreq_default_governor();
Quentin Perret8412b452020-06-29 13:55:00 +05301091 __module_get(gov->owner);
1092 }
1093
Rafael J. Wysockide1df262016-02-05 02:37:42 +01001094 } else {
Quentin Perret8412b452020-06-29 13:55:00 +05301095
Yue Huab05d972019-04-29 15:24:18 +08001096 /* Use the default policy if there is no last_policy. */
1097 if (policy->last_policy) {
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01001098 pol = policy->last_policy;
Quentin Perret8412b452020-06-29 13:55:00 +05301099 } else {
1100 pol = cpufreq_parse_policy(default_governor);
Rafael J. Wysockif5739cb2020-02-26 22:39:27 +01001101 /*
Quentin Perret8412b452020-06-29 13:55:00 +05301102 * In case the default governor is neither "performance"
Rafael J. Wysockif5739cb2020-02-26 22:39:27 +01001103 * nor "powersave", fall back to the initial policy
1104 * value set by the driver.
1105 */
1106 if (pol == CPUFREQ_POLICY_UNKNOWN)
1107 pol = policy->policy;
Yue Huab05d972019-04-29 15:24:18 +08001108 }
Rafael J. Wysockif5739cb2020-02-26 22:39:27 +01001109 if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1110 pol != CPUFREQ_POLICY_POWERSAVE)
1111 return -ENODATA;
Srinivas Pandruvada69030dd12015-12-01 16:52:14 -08001112 }
Yue Huab05d972019-04-29 15:24:18 +08001113
Viresh Kumar8cc46ae2020-06-29 13:54:58 +05301114 ret = cpufreq_set_policy(policy, gov, pol);
1115 if (gov)
1116 module_put(gov->owner);
1117
1118 return ret;
Dave Jones909a6942009-07-08 18:05:42 -04001119}
1120
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001121static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
Viresh Kumarfcf80582013-01-29 14:39:08 +00001122{
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05301123 int ret = 0;
Viresh Kumarfcf80582013-01-29 14:39:08 +00001124
Viresh Kumarbb29ae12015-02-19 17:02:06 +05301125 /* Has this CPU been taken care of already? */
1126 if (cpumask_test_cpu(cpu, policy->cpus))
1127 return 0;
1128
Viresh Kumar49f18562016-02-11 17:31:12 +05301129 down_write(&policy->rwsem);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02001130 if (has_target())
1131 cpufreq_stop_governor(policy);
Viresh Kumarfcf80582013-01-29 14:39:08 +00001132
Viresh Kumarfcf80582013-01-29 14:39:08 +00001133 cpumask_set_cpu(cpu, policy->cpus);
Viresh Kumar2eaa3e22013-02-07 10:55:00 +05301134
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05301135 if (has_target()) {
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01001136 ret = cpufreq_start_governor(policy);
Viresh Kumar49f18562016-02-11 17:31:12 +05301137 if (ret)
Viresh Kumar3de9bde2013-08-06 22:53:13 +05301138 pr_err("%s: Failed to start governor\n", __func__);
Viresh Kumar820c6ca2013-04-22 00:48:03 +02001139 }
Viresh Kumar49f18562016-02-11 17:31:12 +05301140 up_write(&policy->rwsem);
1141 return ret;
Viresh Kumarfcf80582013-01-29 14:39:08 +00001142}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
Viresh Kumarc57b25b2019-07-04 13:06:22 +05301144void refresh_frequency_limits(struct cpufreq_policy *policy)
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301145{
Viresh Kumar67d874c2019-07-08 16:27:52 +05301146 if (!policy_is_inactive(policy)) {
Viresh Kumar67d874c2019-07-08 16:27:52 +05301147 pr_debug("updating policy for CPU %u\n", policy->cpu);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301148
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01001149 cpufreq_set_policy(policy, policy->governor, policy->policy);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301150 }
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301151}
Viresh Kumarc57b25b2019-07-04 13:06:22 +05301152EXPORT_SYMBOL(refresh_frequency_limits);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301153
Viresh Kumar11eb69b2016-02-22 16:36:42 +05301154static void handle_update(struct work_struct *work)
1155{
1156 struct cpufreq_policy *policy =
1157 container_of(work, struct cpufreq_policy, update);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301158
1159 pr_debug("handle_update for cpu %u called\n", policy->cpu);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301160 down_write(&policy->rwsem);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301161 refresh_frequency_limits(policy);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301162 up_write(&policy->rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163}
1164
Viresh Kumar67d874c2019-07-08 16:27:52 +05301165static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1166 void *data)
Srivatsa S. Bhat84148092013-07-30 04:25:10 +05301167{
Viresh Kumar67d874c2019-07-08 16:27:52 +05301168 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301169
Viresh Kumar67d874c2019-07-08 16:27:52 +05301170 schedule_work(&policy->update);
1171 return 0;
1172}
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301173
Viresh Kumar67d874c2019-07-08 16:27:52 +05301174static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1175 void *data)
1176{
1177 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301178
Viresh Kumar67d874c2019-07-08 16:27:52 +05301179 schedule_work(&policy->update);
1180 return 0;
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301181}
1182
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301183static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
Viresh Kumar42f921a2013-12-20 21:26:02 +05301184{
1185 struct kobject *kobj;
1186 struct completion *cmp;
1187
Viresh Kumar87549142015-06-10 02:13:21 +02001188 down_write(&policy->rwsem);
Rafael J. Wysocki1aefc752016-05-31 22:14:44 +02001189 cpufreq_stats_free_table(policy);
Viresh Kumar42f921a2013-12-20 21:26:02 +05301190 kobj = &policy->kobj;
1191 cmp = &policy->kobj_unregister;
Viresh Kumar87549142015-06-10 02:13:21 +02001192 up_write(&policy->rwsem);
Viresh Kumar42f921a2013-12-20 21:26:02 +05301193 kobject_put(kobj);
1194
1195 /*
1196 * We need to make sure that the underlying kobj is
1197 * actually not referenced anymore by anybody before we
1198 * proceed with unloading.
1199 */
1200 pr_debug("waiting for dropping of refcount\n");
1201 wait_for_completion(cmp);
1202 pr_debug("wait complete\n");
1203}
1204
Viresh Kumar67d874c2019-07-08 16:27:52 +05301205static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1206{
1207 struct cpufreq_policy *policy;
1208 struct device *dev = get_cpu_device(cpu);
1209 int ret;
1210
1211 if (!dev)
1212 return NULL;
1213
1214 policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1215 if (!policy)
1216 return NULL;
1217
1218 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1219 goto err_free_policy;
1220
1221 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1222 goto err_free_cpumask;
1223
1224 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1225 goto err_free_rcpumask;
1226
1227 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1228 cpufreq_global_kobject, "policy%u", cpu);
1229 if (ret) {
1230 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1231 /*
1232 * The entire policy object will be freed below, but the extra
1233 * memory allocated for the kobject name needs to be freed by
1234 * releasing the kobject.
1235 */
1236 kobject_put(&policy->kobj);
1237 goto err_free_real_cpus;
1238 }
1239
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001240 freq_constraints_init(&policy->constraints);
1241
Viresh Kumar67d874c2019-07-08 16:27:52 +05301242 policy->nb_min.notifier_call = cpufreq_notifier_min;
1243 policy->nb_max.notifier_call = cpufreq_notifier_max;
1244
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001245 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1246 &policy->nb_min);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301247 if (ret) {
1248 dev_err(dev, "Failed to register MIN QoS notifier: %d (%*pbl)\n",
1249 ret, cpumask_pr_args(policy->cpus));
1250 goto err_kobj_remove;
1251 }
1252
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001253 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1254 &policy->nb_max);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301255 if (ret) {
1256 dev_err(dev, "Failed to register MAX QoS notifier: %d (%*pbl)\n",
1257 ret, cpumask_pr_args(policy->cpus));
1258 goto err_min_qos_notifier;
1259 }
1260
1261 INIT_LIST_HEAD(&policy->policy_list);
1262 init_rwsem(&policy->rwsem);
1263 spin_lock_init(&policy->transition_lock);
1264 init_waitqueue_head(&policy->transition_wait);
1265 init_completion(&policy->kobj_unregister);
1266 INIT_WORK(&policy->update, handle_update);
1267
1268 policy->cpu = cpu;
1269 return policy;
1270
1271err_min_qos_notifier:
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001272 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1273 &policy->nb_min);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301274err_kobj_remove:
1275 cpufreq_policy_put_kobj(policy);
1276err_free_real_cpus:
1277 free_cpumask_var(policy->real_cpus);
1278err_free_rcpumask:
1279 free_cpumask_var(policy->related_cpus);
1280err_free_cpumask:
1281 free_cpumask_var(policy->cpus);
1282err_free_policy:
1283 kfree(policy);
1284
1285 return NULL;
1286}
1287
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301288static void cpufreq_policy_free(struct cpufreq_policy *policy)
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301289{
Viresh Kumar988bed02015-05-08 11:53:45 +05301290 unsigned long flags;
1291 int cpu;
1292
1293 /* Remove policy from list */
1294 write_lock_irqsave(&cpufreq_driver_lock, flags);
1295 list_del(&policy->policy_list);
1296
1297 for_each_cpu(cpu, policy->related_cpus)
1298 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1299 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1300
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001301 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1302 &policy->nb_max);
1303 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1304 &policy->nb_min);
Viresh Kumar6a149032019-07-23 11:44:01 +05301305
Sudeep Holla69410512019-10-18 11:58:15 +01001306 /* Cancel any pending policy->update work before freeing the policy. */
1307 cancel_work_sync(&policy->update);
Viresh Kumar6a149032019-07-23 11:44:01 +05301308
1309 if (policy->max_freq_req) {
1310 /*
1311 * CPUFREQ_CREATE_POLICY notification is sent only after
1312 * successfully adding max_freq_req request.
1313 */
1314 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1315 CPUFREQ_REMOVE_POLICY, policy);
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001316 freq_qos_remove_request(policy->max_freq_req);
Viresh Kumar6a149032019-07-23 11:44:01 +05301317 }
1318
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001319 freq_qos_remove_request(policy->min_freq_req);
Viresh Kumar18c49922019-07-05 16:21:24 +05301320 kfree(policy->min_freq_req);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301321
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301322 cpufreq_policy_put_kobj(policy);
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001323 free_cpumask_var(policy->real_cpus);
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301324 free_cpumask_var(policy->related_cpus);
1325 free_cpumask_var(policy->cpus);
1326 kfree(policy);
1327}
1328
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001329static int cpufreq_online(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330{
Viresh Kumar7f0c0202015-01-02 12:34:32 +05301331 struct cpufreq_policy *policy;
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001332 bool new_policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 unsigned long flags;
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001334 unsigned int j;
1335 int ret;
Ashok Rajc32b6b82005-10-30 14:59:54 -08001336
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001337 pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
Viresh Kumar6eed9402013-08-06 22:53:11 +05301338
Viresh Kumarbb29ae12015-02-19 17:02:06 +05301339 /* Check if this CPU already has a policy to manage it */
Viresh Kumar9104bb22015-05-12 12:22:12 +05301340 policy = per_cpu(cpufreq_cpu_data, cpu);
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001341 if (policy) {
Viresh Kumar9104bb22015-05-12 12:22:12 +05301342 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001343 if (!policy_is_inactive(policy))
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001344 return cpufreq_add_policy_cpu(policy, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001346 /* This is the only online CPU for the policy. Start over. */
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001347 new_policy = false;
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001348 down_write(&policy->rwsem);
1349 policy->cpu = cpu;
1350 policy->governor = NULL;
1351 up_write(&policy->rwsem);
1352 } else {
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001353 new_policy = true;
Rafael J. Wysockia34e63b2015-07-27 23:11:50 +02001354 policy = cpufreq_policy_alloc(cpu);
Rafael J. Wysocki72368d12013-12-27 01:07:11 +01001355 if (!policy)
Rafael J. Wysockid4d854d2015-07-27 23:11:30 +02001356 return -ENOMEM;
Rafael J. Wysocki72368d12013-12-27 01:07:11 +01001357 }
Srivatsa S. Bhat0d66b912013-09-12 01:42:59 +05301358
Viresh Kumar91a12e92019-02-12 16:36:04 +05301359 if (!new_policy && cpufreq_driver->online) {
1360 ret = cpufreq_driver->online(policy);
1361 if (ret) {
1362 pr_debug("%s: %d: initialization failed\n", __func__,
1363 __LINE__);
1364 goto out_exit_policy;
1365 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366
Viresh Kumar91a12e92019-02-12 16:36:04 +05301367 /* Recover policy->cpus using related_cpus */
1368 cpumask_copy(policy->cpus, policy->related_cpus);
1369 } else {
1370 cpumask_copy(policy->cpus, cpumask_of(cpu));
Viresh Kumar643ae6e2013-01-12 05:14:38 +00001371
Viresh Kumar91a12e92019-02-12 16:36:04 +05301372 /*
1373 * Call driver. From then on the cpufreq must be able
1374 * to accept all calls to ->verify and ->setpolicy for this CPU.
1375 */
1376 ret = cpufreq_driver->init(policy);
1377 if (ret) {
1378 pr_debug("%s: %d: initialization failed\n", __func__,
1379 __LINE__);
1380 goto out_free_policy;
1381 }
Viresh Kumard417e062018-02-22 11:29:44 +05301382
Rafael J. Wysocki3e33b132021-06-22 21:11:39 +02001383 /*
1384 * The initialization has succeeded and the policy is online.
1385 * If there is a problem with its frequency table, take it
1386 * offline and drop it.
1387 */
Viresh Kumar91a12e92019-02-12 16:36:04 +05301388 ret = cpufreq_table_validate_and_sort(policy);
1389 if (ret)
Rafael J. Wysocki3e33b132021-06-22 21:11:39 +02001390 goto out_offline_policy;
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001391
Rafael J. Wysocki4d1f3a52015-07-27 23:11:44 +02001392 /* related_cpus should at least include policy->cpus. */
Viresh Kumar0998a032015-10-15 21:35:21 +05301393 cpumask_copy(policy->related_cpus, policy->cpus);
Rafael J. Wysocki4d1f3a52015-07-27 23:11:44 +02001394 }
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001395
Viresh Kumar91a12e92019-02-12 16:36:04 +05301396 down_write(&policy->rwsem);
Viresh Kumar5a7e56a2014-03-04 11:44:00 +08001397 /*
1398 * affected cpus must always be the one, which are online. We aren't
1399 * managing offline cpus here.
1400 */
1401 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1402
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001403 if (new_policy) {
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001404 for_each_cpu(j, policy->related_cpus) {
Viresh Kumar988bed02015-05-08 11:53:45 +05301405 per_cpu(cpufreq_cpu_data, j) = policy;
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001406 add_cpu_dev_symlink(policy, j);
1407 }
Viresh Kumar18c49922019-07-05 16:21:24 +05301408
1409 policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
1410 GFP_KERNEL);
1411 if (!policy->min_freq_req)
1412 goto out_destroy_policy;
1413
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001414 ret = freq_qos_add_request(&policy->constraints,
1415 policy->min_freq_req, FREQ_QOS_MIN,
1416 policy->min);
Viresh Kumar18c49922019-07-05 16:21:24 +05301417 if (ret < 0) {
1418 /*
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001419 * So we don't call freq_qos_remove_request() for an
Viresh Kumar18c49922019-07-05 16:21:24 +05301420 * uninitialized request.
1421 */
1422 kfree(policy->min_freq_req);
1423 policy->min_freq_req = NULL;
Viresh Kumar18c49922019-07-05 16:21:24 +05301424 goto out_destroy_policy;
1425 }
1426
1427 /*
1428 * This must be initialized right here to avoid calling
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001429 * freq_qos_remove_request() on uninitialized request in case
Viresh Kumar18c49922019-07-05 16:21:24 +05301430 * of errors.
1431 */
1432 policy->max_freq_req = policy->min_freq_req + 1;
1433
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001434 ret = freq_qos_add_request(&policy->constraints,
1435 policy->max_freq_req, FREQ_QOS_MAX,
1436 policy->max);
Viresh Kumar18c49922019-07-05 16:21:24 +05301437 if (ret < 0) {
1438 policy->max_freq_req = NULL;
Viresh Kumar18c49922019-07-05 16:21:24 +05301439 goto out_destroy_policy;
1440 }
Viresh Kumar6a149032019-07-23 11:44:01 +05301441
1442 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1443 CPUFREQ_CREATE_POLICY, policy);
Viresh Kumar988bed02015-05-08 11:53:45 +05301444 }
Viresh Kumar652ed952014-01-09 20:38:43 +05301445
Viresh Kumar5ddc6d42019-06-20 08:35:48 +05301446 if (cpufreq_driver->get && has_target()) {
Viresh Kumarda60ce92013-10-03 20:28:30 +05301447 policy->cur = cpufreq_driver->get(policy->cpu);
1448 if (!policy->cur) {
1449 pr_err("%s: ->get() failed\n", __func__);
Viresh Kumard417e062018-02-22 11:29:44 +05301450 goto out_destroy_policy;
Viresh Kumarda60ce92013-10-03 20:28:30 +05301451 }
1452 }
1453
Viresh Kumard3916692013-12-03 11:20:46 +05301454 /*
1455 * Sometimes boot loaders set CPU frequency to a value outside of
1456 * frequency table present with cpufreq core. In such cases CPU might be
1457 * unstable if it has to run on that frequency for long duration of time
1458 * and so its better to set it to a frequency which is specified in
1459 * freq-table. This also makes cpufreq stats inconsistent as
1460 * cpufreq-stats would fail to register because current frequency of CPU
1461 * isn't found in freq-table.
1462 *
1463 * Because we don't want this change to effect boot process badly, we go
1464 * for the next freq which is >= policy->cur ('cur' must be set by now,
1465 * otherwise we will end up setting freq to lowest of the table as 'cur'
1466 * is initialized to zero).
1467 *
1468 * We are passing target-freq as "policy->cur - 1" otherwise
1469 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1470 * equal to target-freq.
1471 */
1472 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1473 && has_target()) {
Viresh Kumar97148d02020-10-13 10:42:47 +05301474 unsigned int old_freq = policy->cur;
1475
Viresh Kumard3916692013-12-03 11:20:46 +05301476 /* Are we running at unknown frequency ? */
Viresh Kumar97148d02020-10-13 10:42:47 +05301477 ret = cpufreq_frequency_table_get_index(policy, old_freq);
Viresh Kumard3916692013-12-03 11:20:46 +05301478 if (ret == -EINVAL) {
Viresh Kumar97148d02020-10-13 10:42:47 +05301479 ret = __cpufreq_driver_target(policy, old_freq - 1,
1480 CPUFREQ_RELATION_L);
Viresh Kumard3916692013-12-03 11:20:46 +05301481
1482 /*
1483 * Reaching here after boot in a few seconds may not
1484 * mean that system will remain stable at "unknown"
1485 * frequency for longer duration. Hence, a BUG_ON().
1486 */
1487 BUG_ON(ret);
Viresh Kumar97148d02020-10-13 10:42:47 +05301488 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n",
1489 __func__, policy->cpu, old_freq, policy->cur);
Viresh Kumard3916692013-12-03 11:20:46 +05301490 }
1491 }
1492
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001493 if (new_policy) {
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001494 ret = cpufreq_add_dev_interface(policy);
Srivatsa S. Bhata82fab22013-07-30 04:24:49 +05301495 if (ret)
Viresh Kumard417e062018-02-22 11:29:44 +05301496 goto out_destroy_policy;
Rafael J. Wysocki1aefc752016-05-31 22:14:44 +02001497
1498 cpufreq_stats_create_table(policy);
Connor O'Brien8bea10e2018-01-31 18:11:57 -08001499 cpufreq_times_create_policy(policy);
Dave Jones8ff69732006-03-05 03:37:23 -05001500
Viresh Kumar988bed02015-05-08 11:53:45 +05301501 write_lock_irqsave(&cpufreq_driver_lock, flags);
1502 list_add(&policy->policy_list, &cpufreq_policy_list);
1503 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1504 }
Viresh Kumar9515f4d2013-08-20 12:08:23 +05301505
Viresh Kumar7f0fa402015-07-08 15:12:16 +05301506 ret = cpufreq_init_policy(policy);
1507 if (ret) {
1508 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1509 __func__, cpu, ret);
Viresh Kumard417e062018-02-22 11:29:44 +05301510 goto out_destroy_policy;
Viresh Kumar08fd8c1c2013-12-24 07:11:01 +05301511 }
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301512
Viresh Kumar4e97b632014-03-04 11:44:01 +08001513 up_write(&policy->rwsem);
Viresh Kumar08fd8c1c2013-12-24 07:11:01 +05301514
Greg Kroah-Hartman038c5b32007-12-17 15:54:39 -04001515 kobject_uevent(&policy->kobj, KOBJ_ADD);
Viresh Kumar7c45cf32014-11-27 06:07:51 +05301516
Viresh Kumar7c45cf32014-11-27 06:07:51 +05301517 /* Callback for handling stuff after policy is ready */
1518 if (cpufreq_driver->ready)
1519 cpufreq_driver->ready(policy);
1520
Daniel Lezcanobcc61562019-06-25 13:32:41 +02001521 if (cpufreq_thermal_control_enabled(cpufreq_driver))
Amit Kucheria5c238a82019-01-30 10:52:01 +05301522 policy->cdev = of_cpufreq_cooling_register(policy);
1523
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02001524 pr_debug("initialization complete\n");
Dave Jones87c32272006-03-29 01:48:37 -05001525
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 return 0;
1527
Viresh Kumard417e062018-02-22 11:29:44 +05301528out_destroy_policy:
Viresh Kumarb24b6472018-02-22 11:29:43 +05301529 for_each_cpu(j, policy->real_cpus)
1530 remove_cpu_dev_symlink(policy, get_cpu_device(j));
1531
Prarit Bhargava7106e022014-09-10 10:12:08 -04001532 up_write(&policy->rwsem);
1533
Rafael J. Wysocki3e33b132021-06-22 21:11:39 +02001534out_offline_policy:
1535 if (cpufreq_driver->offline)
1536 cpufreq_driver->offline(policy);
1537
Viresh Kumard417e062018-02-22 11:29:44 +05301538out_exit_policy:
Viresh Kumarda60ce92013-10-03 20:28:30 +05301539 if (cpufreq_driver->exit)
1540 cpufreq_driver->exit(policy);
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001541
Viresh Kumar8101f992015-07-08 15:12:15 +05301542out_free_policy:
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301543 cpufreq_policy_free(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 return ret;
1545}
1546
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001547/**
1548 * cpufreq_add_dev - the cpufreq interface for a CPU device.
1549 * @dev: CPU device.
1550 * @sif: Subsystem interface structure pointer (not used)
1551 */
1552static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1553{
Rafael J. Wysockia794d612016-04-07 03:31:57 +02001554 struct cpufreq_policy *policy;
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001555 unsigned cpu = dev->id;
Viresh Kumar266198042016-09-12 12:07:05 +05301556 int ret;
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001557
1558 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1559
Viresh Kumar266198042016-09-12 12:07:05 +05301560 if (cpu_online(cpu)) {
1561 ret = cpufreq_online(cpu);
1562 if (ret)
1563 return ret;
1564 }
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001565
Viresh Kumar266198042016-09-12 12:07:05 +05301566 /* Create sysfs link on CPU registration */
Rafael J. Wysockia794d612016-04-07 03:31:57 +02001567 policy = per_cpu(cpufreq_cpu_data, cpu);
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001568 if (policy)
1569 add_cpu_dev_symlink(policy, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001571 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572}
1573
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02001574static int cpufreq_offline(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575{
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05301576 struct cpufreq_policy *policy;
Viresh Kumar69cee712016-02-11 17:31:11 +05301577 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578
Viresh Kumarb8eed8a2013-01-14 13:23:03 +00001579 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580
Viresh Kumar988bed02015-05-08 11:53:45 +05301581 policy = cpufreq_cpu_get_raw(cpu);
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05301582 if (!policy) {
Viresh Kumarb8eed8a2013-01-14 13:23:03 +00001583 pr_debug("%s: No cpu_data found\n", __func__);
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02001584 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586
Viresh Kumar49f18562016-02-11 17:31:12 +05301587 down_write(&policy->rwsem);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02001588 if (has_target())
1589 cpufreq_stop_governor(policy);
Jacob Shin27ecddc2011-04-27 13:32:11 -05001590
Viresh Kumar9591bec2015-06-10 02:20:23 +02001591 cpumask_clear_cpu(cpu, policy->cpus);
Viresh Kumar45732372015-05-12 12:22:34 +05301592
Viresh Kumar9591bec2015-06-10 02:20:23 +02001593 if (policy_is_inactive(policy)) {
1594 if (has_target())
1595 strncpy(policy->last_governor, policy->governor->name,
1596 CPUFREQ_NAME_LEN);
Srinivas Pandruvada69030dd12015-12-01 16:52:14 -08001597 else
1598 policy->last_policy = policy->policy;
Viresh Kumar9591bec2015-06-10 02:20:23 +02001599 } else if (cpu == policy->cpu) {
1600 /* Nominate new CPU */
1601 policy->cpu = cpumask_any(policy->cpus);
1602 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603
Viresh Kumar9591bec2015-06-10 02:20:23 +02001604 /* Start governor again for active policy */
1605 if (!policy_is_inactive(policy)) {
1606 if (has_target()) {
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01001607 ret = cpufreq_start_governor(policy);
Viresh Kumar9591bec2015-06-10 02:20:23 +02001608 if (ret)
1609 pr_err("%s: Failed to start governor\n", __func__);
1610 }
Viresh Kumar69cee712016-02-11 17:31:11 +05301611
Viresh Kumar49f18562016-02-11 17:31:12 +05301612 goto unlock;
Viresh Kumar69cee712016-02-11 17:31:11 +05301613 }
1614
Daniel Lezcanobcc61562019-06-25 13:32:41 +02001615 if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
Amit Kucheria5c238a82019-01-30 10:52:01 +05301616 cpufreq_cooling_unregister(policy->cdev);
1617 policy->cdev = NULL;
1618 }
1619
Viresh Kumar69cee712016-02-11 17:31:11 +05301620 if (cpufreq_driver->stop_cpu)
Dirk Brandewie367dc4a2014-03-19 08:45:53 -07001621 cpufreq_driver->stop_cpu(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02001623 if (has_target())
1624 cpufreq_exit_governor(policy);
Viresh Kumar87549142015-06-10 02:13:21 +02001625
Viresh Kumar87549142015-06-10 02:13:21 +02001626 /*
Viresh Kumar91a12e92019-02-12 16:36:04 +05301627 * Perform the ->offline() during light-weight tear-down, as
1628 * that allows fast recovery when the CPU comes back.
Viresh Kumar87549142015-06-10 02:13:21 +02001629 */
Viresh Kumar91a12e92019-02-12 16:36:04 +05301630 if (cpufreq_driver->offline) {
1631 cpufreq_driver->offline(policy);
1632 } else if (cpufreq_driver->exit) {
Viresh Kumar87549142015-06-10 02:13:21 +02001633 cpufreq_driver->exit(policy);
Srinivas Pandruvada55582bc2015-10-07 13:50:44 -07001634 policy->freq_table = NULL;
1635 }
Viresh Kumar49f18562016-02-11 17:31:12 +05301636
1637unlock:
1638 up_write(&policy->rwsem);
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02001639 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640}
1641
Lee Jonesa9909c212020-07-15 09:26:23 +01001642/*
Viresh Kumar27a862e2013-10-02 14:13:14 +05301643 * cpufreq_remove_dev - remove a CPU device
Srivatsa S. Bhatcedb70a2013-09-07 01:23:09 +05301644 *
1645 * Removes the cpufreq interface for a CPU device.
Srivatsa S. Bhatcedb70a2013-09-07 01:23:09 +05301646 */
Viresh Kumar71db87b2015-07-30 15:04:01 +05301647static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001648{
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001649 unsigned int cpu = dev->id;
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001650 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
Venki Pallipadiec282972007-03-26 12:03:19 -07001651
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001652 if (!policy)
Linus Torvalds1af115d2015-08-31 08:47:40 -07001653 return;
Viresh Kumar87549142015-06-10 02:13:21 +02001654
Viresh Kumar69cee712016-02-11 17:31:11 +05301655 if (cpu_online(cpu))
1656 cpufreq_offline(cpu);
Viresh Kumar87549142015-06-10 02:13:21 +02001657
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001658 cpumask_clear_cpu(cpu, policy->real_cpus);
Viresh Kumar266198042016-09-12 12:07:05 +05301659 remove_cpu_dev_symlink(policy, dev);
Viresh Kumarf344dae2015-11-21 09:06:49 +05301660
Viresh Kumar91a12e92019-02-12 16:36:04 +05301661 if (cpumask_empty(policy->real_cpus)) {
1662 /* We did light-weight exit earlier, do full tear down now */
1663 if (cpufreq_driver->offline)
1664 cpufreq_driver->exit(policy);
1665
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301666 cpufreq_policy_free(policy);
Viresh Kumar91a12e92019-02-12 16:36:04 +05301667 }
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001668}
1669
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670/**
Viresh Kumarbb176f72013-06-19 14:19:33 +05301671 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1672 * in deep trouble.
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301673 * @policy: policy managing CPUs
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674 * @new_freq: CPU frequency the CPU actually runs at
1675 *
Dave Jones29464f22009-01-18 01:37:11 -05001676 * We adjust to current frequency first, and need to clean up later.
1677 * So either call to cpufreq_update_policy() or schedule handle_update()).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001678 */
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301679static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
Gautham R Shenoye08f5f52006-10-26 16:20:58 +05301680 unsigned int new_freq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681{
1682 struct cpufreq_freqs freqs;
Viresh Kumarb43a7ff2013-03-24 11:56:43 +05301683
Joe Perchese837f9b2014-03-11 10:03:00 -07001684 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301685 policy->cur, new_freq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001686
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301687 freqs.old = policy->cur;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688 freqs.new = new_freq;
Viresh Kumarb43a7ff2013-03-24 11:56:43 +05301689
Viresh Kumar8fec0512014-03-24 13:35:45 +05301690 cpufreq_freq_transition_begin(policy, &freqs);
1691 cpufreq_freq_transition_end(policy, &freqs, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001692}
1693
Viresh Kumar59807522019-06-20 08:35:49 +05301694static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1695{
1696 unsigned int new_freq;
1697
1698 new_freq = cpufreq_driver->get(policy->cpu);
1699 if (!new_freq)
1700 return 0;
1701
1702 /*
1703 * If fast frequency switching is used with the given policy, the check
1704 * against policy->cur is pointless, so skip it in that case.
1705 */
1706 if (policy->fast_switch_enabled || !has_target())
1707 return new_freq;
1708
1709 if (policy->cur != new_freq) {
1710 cpufreq_out_of_sync(policy, new_freq);
1711 if (update)
1712 schedule_work(&policy->update);
1713 }
1714
1715 return new_freq;
1716}
1717
Dave Jones32ee8c32006-02-28 00:43:23 -05001718/**
Dhaval Giani4ab70df2006-12-13 14:49:15 +05301719 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001720 * @cpu: CPU number
1721 *
1722 * This is the last known freq, without actually getting it from the driver.
1723 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1724 */
1725unsigned int cpufreq_quick_get(unsigned int cpu)
1726{
Dirk Brandewie9e21ba82013-02-06 09:02:08 -08001727 struct cpufreq_policy *policy;
Gautham R Shenoye08f5f52006-10-26 16:20:58 +05301728 unsigned int ret_freq = 0;
Richard Cochranc75361c2016-03-11 09:43:07 +01001729 unsigned long flags;
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001730
Richard Cochranc75361c2016-03-11 09:43:07 +01001731 read_lock_irqsave(&cpufreq_driver_lock, flags);
1732
1733 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1734 ret_freq = cpufreq_driver->get(cpu);
1735 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1736 return ret_freq;
1737 }
1738
1739 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Dirk Brandewie9e21ba82013-02-06 09:02:08 -08001740
1741 policy = cpufreq_cpu_get(cpu);
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001742 if (policy) {
Gautham R Shenoye08f5f52006-10-26 16:20:58 +05301743 ret_freq = policy->cur;
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001744 cpufreq_cpu_put(policy);
1745 }
1746
Dave Jones4d34a672008-02-07 16:33:49 -05001747 return ret_freq;
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001748}
1749EXPORT_SYMBOL(cpufreq_quick_get);
1750
Jesse Barnes3d737102011-06-28 10:59:12 -07001751/**
1752 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1753 * @cpu: CPU number
1754 *
1755 * Just return the max possible frequency for a given CPU.
1756 */
1757unsigned int cpufreq_quick_get_max(unsigned int cpu)
1758{
1759 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1760 unsigned int ret_freq = 0;
1761
1762 if (policy) {
1763 ret_freq = policy->max;
1764 cpufreq_cpu_put(policy);
1765 }
1766
1767 return ret_freq;
1768}
1769EXPORT_SYMBOL(cpufreq_quick_get_max);
1770
Ionela Voinescubbce8ea2020-03-05 09:06:25 +00001771/**
1772 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1773 * @cpu: CPU number
1774 *
1775 * The default return value is the max_freq field of cpuinfo.
1776 */
1777__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1778{
1779 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1780 unsigned int ret_freq = 0;
1781
1782 if (policy) {
1783 ret_freq = policy->cpuinfo.max_freq;
1784 cpufreq_cpu_put(policy);
1785 }
1786
1787 return ret_freq;
1788}
1789EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1790
Viresh Kumard92d50a2015-01-02 12:34:29 +05301791static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792{
Yue Hu4db7c342019-04-19 14:27:58 +08001793 if (unlikely(policy_is_inactive(policy)))
Viresh Kumar59807522019-06-20 08:35:49 +05301794 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795
Viresh Kumar59807522019-06-20 08:35:49 +05301796 return cpufreq_verify_current_freq(policy, true);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001797}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001799/**
1800 * cpufreq_get - get the current CPU frequency (in kHz)
1801 * @cpu: CPU number
1802 *
1803 * Get the CPU current (static) CPU frequency
1804 */
1805unsigned int cpufreq_get(unsigned int cpu)
1806{
Aaron Plattner999976e2014-03-04 12:42:15 -08001807 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001808 unsigned int ret_freq = 0;
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001809
Aaron Plattner999976e2014-03-04 12:42:15 -08001810 if (policy) {
1811 down_read(&policy->rwsem);
Yue Hu4db7c342019-04-19 14:27:58 +08001812 if (cpufreq_driver->get)
1813 ret_freq = __cpufreq_get(policy);
Aaron Plattner999976e2014-03-04 12:42:15 -08001814 up_read(&policy->rwsem);
Viresh Kumar26ca8692013-09-20 22:37:31 +05301815
Aaron Plattner999976e2014-03-04 12:42:15 -08001816 cpufreq_cpu_put(policy);
1817 }
Viresh Kumar6eed9402013-08-06 22:53:11 +05301818
Dave Jones4d34a672008-02-07 16:33:49 -05001819 return ret_freq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820}
1821EXPORT_SYMBOL(cpufreq_get);
1822
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001823static struct subsys_interface cpufreq_interface = {
1824 .name = "cpufreq",
1825 .subsys = &cpu_subsys,
1826 .add_dev = cpufreq_add_dev,
1827 .remove_dev = cpufreq_remove_dev,
Rafael J. Wysockie00e56d2011-03-23 22:16:32 +01001828};
1829
Viresh Kumare28867e2014-03-04 11:00:27 +08001830/*
1831 * In case platform wants some specific frequency to be configured
1832 * during suspend..
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001833 */
Viresh Kumare28867e2014-03-04 11:00:27 +08001834int cpufreq_generic_suspend(struct cpufreq_policy *policy)
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001835{
Viresh Kumare28867e2014-03-04 11:00:27 +08001836 int ret;
Dave Jones4bc5d342009-08-04 14:03:25 -04001837
Viresh Kumare28867e2014-03-04 11:00:27 +08001838 if (!policy->suspend_freq) {
Bartlomiej Zolnierkiewicz201f3712015-09-08 18:41:02 +02001839 pr_debug("%s: suspend_freq not defined\n", __func__);
1840 return 0;
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001841 }
1842
Viresh Kumare28867e2014-03-04 11:00:27 +08001843 pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1844 policy->suspend_freq);
1845
1846 ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1847 CPUFREQ_RELATION_H);
1848 if (ret)
1849 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1850 __func__, policy->suspend_freq, ret);
1851
Dave Jonesc9060492008-02-07 16:32:18 -05001852 return ret;
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001853}
Viresh Kumare28867e2014-03-04 11:00:27 +08001854EXPORT_SYMBOL(cpufreq_generic_suspend);
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001855
1856/**
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001857 * cpufreq_suspend() - Suspend CPUFreq governors
Linus Torvalds1da177e2005-04-16 15:20:36 -07001858 *
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001859 * Called during system wide Suspend/Hibernate cycles for suspending governors
1860 * as some platforms can't change frequency after this point in suspend cycle.
1861 * Because some of the devices (like: i2c, regulators, etc) they use for
1862 * changing frequency are suspended quickly after this point.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 */
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001864void cpufreq_suspend(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865{
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05301866 struct cpufreq_policy *policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001867
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001868 if (!cpufreq_driver)
Rafael J. Wysockie00e56d2011-03-23 22:16:32 +01001869 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001870
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001871 if (!has_target() && !cpufreq_driver->suspend)
Viresh Kumarb1b12ba2014-09-30 09:33:17 +05301872 goto suspend;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001874 pr_debug("%s: Suspending Governors\n", __func__);
1875
Viresh Kumarf9637352015-05-12 12:20:11 +05301876 for_each_active_policy(policy) {
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001877 if (has_target()) {
1878 down_write(&policy->rwsem);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02001879 cpufreq_stop_governor(policy);
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001880 up_write(&policy->rwsem);
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001881 }
1882
1883 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
Florian Fainellie9a7cc12019-08-21 16:16:32 -07001884 pr_err("%s: Failed to suspend driver: %s\n", __func__,
1885 cpufreq_driver->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001886 }
Viresh Kumarb1b12ba2014-09-30 09:33:17 +05301887
1888suspend:
1889 cpufreq_suspended = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001890}
1891
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892/**
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001893 * cpufreq_resume() - Resume CPUFreq governors
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894 *
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001895 * Called during system wide Suspend/Hibernate cycle for resuming governors that
1896 * are suspended with cpufreq_suspend().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897 */
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001898void cpufreq_resume(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900 struct cpufreq_policy *policy;
Viresh Kumar49f18562016-02-11 17:31:12 +05301901 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001903 if (!cpufreq_driver)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 return;
1905
Bo Yan703cbaa2018-01-23 13:57:55 -08001906 if (unlikely(!cpufreq_suspended))
1907 return;
1908
Lan Tianyu8e304442014-09-18 15:03:07 +08001909 cpufreq_suspended = false;
1910
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001911 if (!has_target() && !cpufreq_driver->resume)
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001912 return;
1913
1914 pr_debug("%s: Resuming Governors\n", __func__);
1915
Viresh Kumarf9637352015-05-12 12:20:11 +05301916 for_each_active_policy(policy) {
Viresh Kumar49f18562016-02-11 17:31:12 +05301917 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
Viresh Kumar0c5aa402014-03-24 12:30:29 +05301918 pr_err("%s: Failed to resume driver: %p\n", __func__,
1919 policy);
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001920 } else if (has_target()) {
Viresh Kumar49f18562016-02-11 17:31:12 +05301921 down_write(&policy->rwsem);
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01001922 ret = cpufreq_start_governor(policy);
Viresh Kumar49f18562016-02-11 17:31:12 +05301923 up_write(&policy->rwsem);
1924
1925 if (ret)
1926 pr_err("%s: Failed to start governor for policy: %p\n",
1927 __func__, policy);
1928 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001929 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001930}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001931
Borislav Petkov9d950462013-01-20 10:24:28 +00001932/**
Rafael J. Wysockia62f68f2020-10-23 17:35:46 +02001933 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
1934 * @flags: Flags to test against the current cpufreq driver's flags.
1935 *
1936 * Assumes that the driver is there, so callers must ensure that this is the
1937 * case.
1938 */
1939bool cpufreq_driver_test_flags(u16 flags)
1940{
1941 return !!(cpufreq_driver->flags & flags);
1942}
1943
1944/**
Borislav Petkov9d950462013-01-20 10:24:28 +00001945 * cpufreq_get_current_driver - return current driver's name
1946 *
1947 * Return the name string of the currently loaded cpufreq driver
1948 * or NULL, if none.
1949 */
1950const char *cpufreq_get_current_driver(void)
1951{
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001952 if (cpufreq_driver)
1953 return cpufreq_driver->name;
1954
1955 return NULL;
Borislav Petkov9d950462013-01-20 10:24:28 +00001956}
1957EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001958
Thomas Petazzoni51315cd2014-10-19 11:30:27 +02001959/**
1960 * cpufreq_get_driver_data - return current driver data
1961 *
1962 * Return the private data of the currently loaded cpufreq
1963 * driver, or NULL if no cpufreq driver is loaded.
1964 */
1965void *cpufreq_get_driver_data(void)
1966{
1967 if (cpufreq_driver)
1968 return cpufreq_driver->driver_data;
1969
1970 return NULL;
1971}
1972EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1973
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974/*********************************************************************
1975 * NOTIFIER LISTS INTERFACE *
1976 *********************************************************************/
1977
1978/**
1979 * cpufreq_register_notifier - register a driver with cpufreq
1980 * @nb: notifier function to register
1981 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1982 *
Dave Jones32ee8c32006-02-28 00:43:23 -05001983 * Add a driver to one of two lists: either a list of drivers that
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984 * are notified about clock rate changes (once before and once after
1985 * the transition), or a list of drivers that are notified about
1986 * changes in cpufreq policy.
1987 *
1988 * This function may sleep, and has the same return conditions as
Alan Sterne041c682006-03-27 01:16:30 -08001989 * blocking_notifier_chain_register.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001990 */
1991int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1992{
1993 int ret;
1994
Dirk Brandewied5aaffa2013-01-17 16:22:21 +00001995 if (cpufreq_disabled())
1996 return -EINVAL;
1997
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998 switch (list) {
1999 case CPUFREQ_TRANSITION_NOTIFIER:
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002000 mutex_lock(&cpufreq_fast_switch_lock);
2001
2002 if (cpufreq_fast_switch_count > 0) {
2003 mutex_unlock(&cpufreq_fast_switch_lock);
2004 return -EBUSY;
2005 }
Alan Sternb4dfdbb2006-10-04 02:17:06 -07002006 ret = srcu_notifier_chain_register(
Alan Sterne041c682006-03-27 01:16:30 -08002007 &cpufreq_transition_notifier_list, nb);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002008 if (!ret)
2009 cpufreq_fast_switch_count--;
2010
2011 mutex_unlock(&cpufreq_fast_switch_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002012 break;
2013 case CPUFREQ_POLICY_NOTIFIER:
Alan Sterne041c682006-03-27 01:16:30 -08002014 ret = blocking_notifier_chain_register(
2015 &cpufreq_policy_notifier_list, nb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002016 break;
2017 default:
2018 ret = -EINVAL;
2019 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002020
2021 return ret;
2022}
2023EXPORT_SYMBOL(cpufreq_register_notifier);
2024
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025/**
2026 * cpufreq_unregister_notifier - unregister a driver with cpufreq
2027 * @nb: notifier block to be unregistered
Viresh Kumarbb176f72013-06-19 14:19:33 +05302028 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029 *
2030 * Remove a driver from the CPU frequency notifier list.
2031 *
2032 * This function may sleep, and has the same return conditions as
Alan Sterne041c682006-03-27 01:16:30 -08002033 * blocking_notifier_chain_unregister.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034 */
2035int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2036{
2037 int ret;
2038
Dirk Brandewied5aaffa2013-01-17 16:22:21 +00002039 if (cpufreq_disabled())
2040 return -EINVAL;
2041
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042 switch (list) {
2043 case CPUFREQ_TRANSITION_NOTIFIER:
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002044 mutex_lock(&cpufreq_fast_switch_lock);
2045
Alan Sternb4dfdbb2006-10-04 02:17:06 -07002046 ret = srcu_notifier_chain_unregister(
Alan Sterne041c682006-03-27 01:16:30 -08002047 &cpufreq_transition_notifier_list, nb);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002048 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2049 cpufreq_fast_switch_count++;
2050
2051 mutex_unlock(&cpufreq_fast_switch_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002052 break;
2053 case CPUFREQ_POLICY_NOTIFIER:
Alan Sterne041c682006-03-27 01:16:30 -08002054 ret = blocking_notifier_chain_unregister(
2055 &cpufreq_policy_notifier_list, nb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056 break;
2057 default:
2058 ret = -EINVAL;
2059 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002060
2061 return ret;
2062}
2063EXPORT_SYMBOL(cpufreq_unregister_notifier);
2064
2065
2066/*********************************************************************
2067 * GOVERNORS *
2068 *********************************************************************/
2069
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002070/**
2071 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2072 * @policy: cpufreq policy to switch the frequency for.
2073 * @target_freq: New frequency to set (may be approximate).
2074 *
2075 * Carry out a fast frequency switch without sleeping.
2076 *
2077 * The driver's ->fast_switch() callback invoked by this function must be
2078 * suitable for being called from within RCU-sched read-side critical sections
2079 * and it is expected to select the minimum available frequency greater than or
2080 * equal to @target_freq (CPUFREQ_RELATION_L).
2081 *
2082 * This function must not be called if policy->fast_switch_enabled is unset.
2083 *
2084 * Governors calling this function must guarantee that it will never be invoked
2085 * twice in parallel for the same policy and that it will never be called in
2086 * parallel with either ->target() or ->target_index() for the same policy.
2087 *
Viresh Kumar209887e2017-08-09 10:21:46 +05302088 * Returns the actual frequency set for the CPU.
2089 *
2090 * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2091 * error condition, the hardware configuration must be preserved.
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002092 */
2093unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2094 unsigned int target_freq)
2095{
Ionela Voinescu1a0419b2020-09-01 21:55:46 +01002096 unsigned int freq;
Viresh Kumar08d8c652020-10-05 13:26:05 +05302097 int cpu;
Connor O'Brien0d084d72019-02-08 12:30:41 -08002098
Rafael J. Wysockib9af6942016-06-01 22:36:26 +02002099 target_freq = clamp_val(target_freq, policy->min, policy->max);
Ionela Voinescu1a0419b2020-09-01 21:55:46 +01002100 freq = cpufreq_driver->fast_switch(policy, target_freq);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002101
Viresh Kumar96f60cd2020-10-05 13:26:04 +05302102 if (!freq)
2103 return 0;
Connor O'Brien0d084d72019-02-08 12:30:41 -08002104
Viresh Kumar08d8c652020-10-05 13:26:05 +05302105 policy->cur = freq;
Ionela Voinescu1a0419b2020-09-01 21:55:46 +01002106 arch_set_freq_scale(policy->related_cpus, freq,
2107 policy->cpuinfo.max_freq);
Viresh Kumar96f60cd2020-10-05 13:26:04 +05302108 cpufreq_stats_record_transition(policy, freq);
Pavankumar Kondeti2d450012021-03-05 04:31:50 +05302109 trace_android_rvh_cpufreq_transition(policy);
Ionela Voinescu1a0419b2020-09-01 21:55:46 +01002110
Viresh Kumar08d8c652020-10-05 13:26:05 +05302111 if (trace_cpu_frequency_enabled()) {
2112 for_each_cpu(cpu, policy->cpus)
2113 trace_cpu_frequency(freq, cpu);
2114 }
2115
Ionela Voinescu1a0419b2020-09-01 21:55:46 +01002116 return freq;
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002117}
2118EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2119
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302120/* Must set freqs->new to intermediate frequency */
2121static int __target_intermediate(struct cpufreq_policy *policy,
2122 struct cpufreq_freqs *freqs, int index)
2123{
2124 int ret;
2125
2126 freqs->new = cpufreq_driver->get_intermediate(policy, index);
2127
2128 /* We don't need to switch to intermediate freq */
2129 if (!freqs->new)
2130 return 0;
2131
2132 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2133 __func__, policy->cpu, freqs->old, freqs->new);
2134
2135 cpufreq_freq_transition_begin(policy, freqs);
2136 ret = cpufreq_driver->target_intermediate(policy, index);
2137 cpufreq_freq_transition_end(policy, freqs, ret);
2138
2139 if (ret)
2140 pr_err("%s: Failed to change to intermediate frequency: %d\n",
2141 __func__, ret);
2142
2143 return ret;
2144}
2145
Viresh Kumar23727842016-06-03 10:58:50 +05302146static int __target_index(struct cpufreq_policy *policy, int index)
Viresh Kumar8d657752014-05-21 14:29:29 +05302147{
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302148 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2149 unsigned int intermediate_freq = 0;
Viresh Kumar23727842016-06-03 10:58:50 +05302150 unsigned int newfreq = policy->freq_table[index].frequency;
Viresh Kumar8d657752014-05-21 14:29:29 +05302151 int retval = -EINVAL;
2152 bool notify;
2153
Viresh Kumar23727842016-06-03 10:58:50 +05302154 if (newfreq == policy->cur)
2155 return 0;
2156
Viresh Kumar8d657752014-05-21 14:29:29 +05302157 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
Viresh Kumar8d657752014-05-21 14:29:29 +05302158 if (notify) {
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302159 /* Handle switching to intermediate frequency */
2160 if (cpufreq_driver->get_intermediate) {
2161 retval = __target_intermediate(policy, &freqs, index);
2162 if (retval)
2163 return retval;
Viresh Kumar8d657752014-05-21 14:29:29 +05302164
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302165 intermediate_freq = freqs.new;
2166 /* Set old freq to intermediate */
2167 if (intermediate_freq)
2168 freqs.old = freqs.new;
2169 }
2170
Viresh Kumar23727842016-06-03 10:58:50 +05302171 freqs.new = newfreq;
Viresh Kumar8d657752014-05-21 14:29:29 +05302172 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2173 __func__, policy->cpu, freqs.old, freqs.new);
2174
2175 cpufreq_freq_transition_begin(policy, &freqs);
2176 }
2177
2178 retval = cpufreq_driver->target_index(policy, index);
2179 if (retval)
2180 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2181 retval);
2182
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302183 if (notify) {
Viresh Kumar8d657752014-05-21 14:29:29 +05302184 cpufreq_freq_transition_end(policy, &freqs, retval);
2185
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302186 /*
2187 * Failed after setting to intermediate freq? Driver should have
2188 * reverted back to initial frequency and so should we. Check
2189 * here for intermediate_freq instead of get_intermediate, in
Shailendra Verma58405af2015-05-22 22:48:22 +05302190 * case we haven't switched to intermediate freq at all.
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302191 */
2192 if (unlikely(retval && intermediate_freq)) {
2193 freqs.old = intermediate_freq;
2194 freqs.new = policy->restore_freq;
2195 cpufreq_freq_transition_begin(policy, &freqs);
2196 cpufreq_freq_transition_end(policy, &freqs, 0);
2197 }
2198 }
2199
Viresh Kumar8d657752014-05-21 14:29:29 +05302200 return retval;
2201}
2202
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203int __cpufreq_driver_target(struct cpufreq_policy *policy,
2204 unsigned int target_freq,
2205 unsigned int relation)
2206{
Viresh Kumar72499242012-10-31 01:28:21 +01002207 unsigned int old_target_freq = target_freq;
Viresh Kumard218ed72016-06-03 10:58:51 +05302208 int index;
Ashok Rajc32b6b82005-10-30 14:59:54 -08002209
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002210 if (cpufreq_disabled())
2211 return -ENODEV;
2212
Viresh Kumar72499242012-10-31 01:28:21 +01002213 /* Make sure that target_freq is within supported range */
Viresh Kumar910c6e82016-05-26 11:27:24 +05302214 target_freq = clamp_val(target_freq, policy->min, policy->max);
Viresh Kumar72499242012-10-31 01:28:21 +01002215
2216 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
Joe Perchese837f9b2014-03-11 10:03:00 -07002217 policy->cpu, target_freq, relation, old_target_freq);
Viresh Kumar5a1c0222012-10-31 01:28:15 +01002218
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302219 /*
2220 * This might look like a redundant call as we are checking it again
2221 * after finding index. But it is left intentionally for cases where
2222 * exactly same freq is called again and so we can save on few function
2223 * calls.
2224 */
Rafael J. Wysocki1c534352020-10-23 17:35:19 +02002225 if (target_freq == policy->cur &&
2226 !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
Viresh Kumar5a1c0222012-10-31 01:28:15 +01002227 return 0;
2228
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302229 /* Save last value to restore later on errors */
2230 policy->restore_freq = policy->cur;
2231
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002232 if (cpufreq_driver->target)
Rafael J. Wysocki6019d232016-02-26 00:03:01 +01002233 return cpufreq_driver->target(policy, target_freq, relation);
Ashok Raj90d45d12005-11-08 21:34:24 -08002234
Rafael J. Wysocki6019d232016-02-26 00:03:01 +01002235 if (!cpufreq_driver->target_index)
2236 return -EINVAL;
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302237
Viresh Kumard218ed72016-06-03 10:58:51 +05302238 index = cpufreq_frequency_table_target(policy, target_freq, relation);
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302239
Viresh Kumar23727842016-06-03 10:58:50 +05302240 return __target_index(policy, index);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002241}
2242EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2243
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244int cpufreq_driver_target(struct cpufreq_policy *policy,
2245 unsigned int target_freq,
2246 unsigned int relation)
2247{
Colin Ian King62c23a82019-08-13 13:21:21 +01002248 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002249
viresh kumarad7722d2013-10-18 19:10:15 +05302250 down_write(&policy->rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002251
2252 ret = __cpufreq_driver_target(policy, target_freq, relation);
2253
viresh kumarad7722d2013-10-18 19:10:15 +05302254 up_write(&policy->rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255
Linus Torvalds1da177e2005-04-16 15:20:36 -07002256 return ret;
2257}
2258EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2259
Rafael J. Wysockide1df262016-02-05 02:37:42 +01002260__weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2261{
2262 return NULL;
2263}
2264
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002265static int cpufreq_init_governor(struct cpufreq_policy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002266{
Dave Jonescc993ca2005-07-28 09:43:56 -07002267 int ret;
Thomas Renninger6afde102007-10-02 13:28:13 -07002268
Viresh Kumar2f0aea92014-03-04 11:00:26 +08002269 /* Don't start any governor operations if we are entering suspend */
2270 if (cpufreq_suspended)
2271 return 0;
Ethan Zhaocb57720b2014-12-18 15:28:19 +09002272 /*
2273 * Governor might not be initiated here if ACPI _PPC changed
2274 * notification happened, so check it.
2275 */
2276 if (!policy->governor)
2277 return -EINVAL;
Viresh Kumar2f0aea92014-03-04 11:00:26 +08002278
Viresh Kumared4676e2017-07-19 15:42:46 +05302279 /* Platform doesn't want dynamic frequency switching ? */
Rafael J. Wysocki9a2a9eb2020-11-10 18:25:57 +01002280 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
Viresh Kumarfc4c7092017-07-19 15:42:49 +05302281 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
Rafael J. Wysockide1df262016-02-05 02:37:42 +01002282 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2283
2284 if (gov) {
Viresh Kumarfe829ed2017-07-19 15:42:48 +05302285 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
Joe Perchese837f9b2014-03-11 10:03:00 -07002286 policy->governor->name, gov->name);
Thomas Renninger6afde102007-10-02 13:28:13 -07002287 policy->governor = gov;
Rafael J. Wysockide1df262016-02-05 02:37:42 +01002288 } else {
2289 return -EINVAL;
Thomas Renninger6afde102007-10-02 13:28:13 -07002290 }
Thomas Renninger1c256242007-10-02 13:28:12 -07002291 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002292
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002293 if (!try_module_get(policy->governor->owner))
2294 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002295
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002296 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
Xiaoguang Chen95731eb2013-06-19 15:00:07 +08002297
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002298 if (policy->governor->init) {
2299 ret = policy->governor->init(policy);
2300 if (ret) {
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02002301 module_put(policy->governor->owner);
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002302 return ret;
2303 }
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02002304 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002305
Rafael J. Wysockiea9364b2020-11-10 18:26:37 +01002306 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2307
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002308 return 0;
2309}
2310
2311static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2312{
2313 if (cpufreq_suspended || !policy->governor)
2314 return;
2315
2316 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2317
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002318 if (policy->governor->exit)
2319 policy->governor->exit(policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002320
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002321 module_put(policy->governor->owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002322}
2323
Rafael J. Wysockif6ebbcf2020-08-06 14:03:55 +02002324int cpufreq_start_governor(struct cpufreq_policy *policy)
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002325{
2326 int ret;
2327
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002328 if (cpufreq_suspended)
2329 return 0;
2330
2331 if (!policy->governor)
2332 return -EINVAL;
2333
2334 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2335
Viresh Kumar407d0ff2019-06-20 08:35:46 +05302336 if (cpufreq_driver->get)
Viresh Kumar59807522019-06-20 08:35:49 +05302337 cpufreq_verify_current_freq(policy, false);
Rafael J. Wysocki3bbf8fe2016-03-21 15:47:48 +01002338
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002339 if (policy->governor->start) {
2340 ret = policy->governor->start(policy);
2341 if (ret)
2342 return ret;
2343 }
Rafael J. Wysockid6ff44d2016-05-14 00:59:27 +02002344
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002345 if (policy->governor->limits)
2346 policy->governor->limits(policy);
2347
Rafael J. Wysockid6ff44d2016-05-14 00:59:27 +02002348 return 0;
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002349}
2350
Rafael J. Wysockif6ebbcf2020-08-06 14:03:55 +02002351void cpufreq_stop_governor(struct cpufreq_policy *policy)
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002352{
2353 if (cpufreq_suspended || !policy->governor)
2354 return;
2355
2356 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2357
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002358 if (policy->governor->stop)
2359 policy->governor->stop(policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002360}
2361
2362static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2363{
2364 if (cpufreq_suspended || !policy->governor)
2365 return;
2366
2367 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2368
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002369 if (policy->governor->limits)
2370 policy->governor->limits(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002371}
2372
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373int cpufreq_register_governor(struct cpufreq_governor *governor)
2374{
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002375 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002376
2377 if (!governor)
2378 return -EINVAL;
2379
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002380 if (cpufreq_disabled())
2381 return -ENODEV;
2382
akpm@osdl.org3fc54d32006-01-13 15:54:22 -08002383 mutex_lock(&cpufreq_governor_mutex);
Dave Jones32ee8c32006-02-28 00:43:23 -05002384
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002385 err = -EBUSY;
Viresh Kumar42f91fa2015-01-02 12:34:26 +05302386 if (!find_governor(governor->name)) {
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002387 err = 0;
2388 list_add(&governor->governor_list, &cpufreq_governor_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002390
Dave Jones32ee8c32006-02-28 00:43:23 -05002391 mutex_unlock(&cpufreq_governor_mutex);
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002392 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393}
2394EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2395
Linus Torvalds1da177e2005-04-16 15:20:36 -07002396void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2397{
Viresh Kumar45732372015-05-12 12:22:34 +05302398 struct cpufreq_policy *policy;
2399 unsigned long flags;
Prarit Bhargava90e41ba2009-11-12 09:18:46 -05002400
Linus Torvalds1da177e2005-04-16 15:20:36 -07002401 if (!governor)
2402 return;
2403
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002404 if (cpufreq_disabled())
2405 return;
2406
Viresh Kumar45732372015-05-12 12:22:34 +05302407 /* clear last_governor for all inactive policies */
2408 read_lock_irqsave(&cpufreq_driver_lock, flags);
2409 for_each_inactive_policy(policy) {
Viresh Kumar18bf3a12015-05-12 12:22:51 +05302410 if (!strcmp(policy->last_governor, governor->name)) {
2411 policy->governor = NULL;
Viresh Kumar45732372015-05-12 12:22:34 +05302412 strcpy(policy->last_governor, "\0");
Viresh Kumar18bf3a12015-05-12 12:22:51 +05302413 }
Prarit Bhargava90e41ba2009-11-12 09:18:46 -05002414 }
Viresh Kumar45732372015-05-12 12:22:34 +05302415 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Prarit Bhargava90e41ba2009-11-12 09:18:46 -05002416
akpm@osdl.org3fc54d32006-01-13 15:54:22 -08002417 mutex_lock(&cpufreq_governor_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418 list_del(&governor->governor_list);
akpm@osdl.org3fc54d32006-01-13 15:54:22 -08002419 mutex_unlock(&cpufreq_governor_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420}
2421EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2422
2423
Linus Torvalds1da177e2005-04-16 15:20:36 -07002424/*********************************************************************
2425 * POLICY INTERFACE *
2426 *********************************************************************/
2427
2428/**
2429 * cpufreq_get_policy - get the current cpufreq_policy
Dave Jones29464f22009-01-18 01:37:11 -05002430 * @policy: struct cpufreq_policy into which the current cpufreq_policy
2431 * is written
Lee Jonesa9909c212020-07-15 09:26:23 +01002432 * @cpu: CPU to find the policy for
Linus Torvalds1da177e2005-04-16 15:20:36 -07002433 *
2434 * Reads the current cpufreq policy.
2435 */
2436int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2437{
2438 struct cpufreq_policy *cpu_policy;
2439 if (!policy)
2440 return -EINVAL;
2441
2442 cpu_policy = cpufreq_cpu_get(cpu);
2443 if (!cpu_policy)
2444 return -EINVAL;
2445
Viresh Kumard5b73cd2013-08-06 22:53:06 +05302446 memcpy(policy, cpu_policy, sizeof(*policy));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002447
2448 cpufreq_cpu_put(cpu_policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449 return 0;
2450}
2451EXPORT_SYMBOL(cpufreq_get_policy);
2452
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002453/**
2454 * cpufreq_set_policy - Modify cpufreq policy parameters.
2455 * @policy: Policy object to modify.
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002456 * @new_gov: Policy governor pointer.
2457 * @new_pol: Policy value (for drivers with built-in governors).
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002458 *
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002459 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2460 * limits to be set for the policy, update @policy with the verified limits
2461 * values and either invoke the driver's ->setpolicy() callback (if present) or
2462 * carry out a governor update for @policy. That is, run the current governor's
2463 * ->limits() callback (if @new_gov points to the same object as the one in
2464 * @policy) or replace the governor for @policy with @new_gov.
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002465 *
2466 * The cpuinfo part of @policy is not updated by this function.
Arjan van de Ven153d7f32006-07-26 15:40:07 +02002467 */
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002468static int cpufreq_set_policy(struct cpufreq_policy *policy,
2469 struct cpufreq_governor *new_gov,
2470 unsigned int new_pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002471{
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002472 struct cpufreq_policy_data new_data;
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002473 struct cpufreq_governor *old_gov;
2474 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002475
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002476 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2477 new_data.freq_table = policy->freq_table;
2478 new_data.cpu = policy->cpu;
Pan Xinhuifba95732015-07-30 18:10:40 +08002479 /*
Viresh Kumar67d874c2019-07-08 16:27:52 +05302480 * PM QoS framework collects all the requests from users and provide us
2481 * the final aggregated value here.
2482 */
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002483 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2484 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
2485
2486 pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2487 new_data.cpu, new_data.min, new_data.max);
Mattia Dongili9c9a43e2006-07-05 23:12:20 +02002488
Viresh Kumar737ffb22019-10-22 15:47:57 +05302489 /*
2490 * Verify that the CPU speed can be set within these limits and make sure
2491 * that min <= max.
2492 */
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002493 ret = cpufreq_driver->verify(&new_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002494 if (ret)
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002495 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002496
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002497 policy->min = new_data.min;
2498 policy->max = new_data.max;
Ruchi Kandoi601b2182018-07-24 10:35:44 -07002499 trace_cpu_frequency_limits(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500
Steve Mucklee3c06232016-07-13 13:25:25 -07002501 policy->cached_target_freq = UINT_MAX;
2502
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002503 pr_debug("new min and max freqs are %u - %u kHz\n",
Joe Perchese837f9b2014-03-11 10:03:00 -07002504 policy->min, policy->max);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002506 if (cpufreq_driver->setpolicy) {
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002507 policy->policy = new_pol;
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002508 pr_debug("setting range\n");
Rafael J. Wysocki167a38d2019-02-20 00:26:30 +01002509 return cpufreq_driver->setpolicy(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002510 }
2511
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002512 if (new_gov == policy->governor) {
Rafael J. Wysocki2bb40592019-02-20 00:25:18 +01002513 pr_debug("governor limits update\n");
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002514 cpufreq_governor_limits(policy);
Rafael J. Wysockid6ff44d2016-05-14 00:59:27 +02002515 return 0;
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002516 }
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002517
2518 pr_debug("governor switch\n");
2519
2520 /* save old, working values */
2521 old_gov = policy->governor;
2522 /* end old governor */
2523 if (old_gov) {
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02002524 cpufreq_stop_governor(policy);
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02002525 cpufreq_exit_governor(policy);
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002526 }
2527
2528 /* start new governor */
Rafael J. Wysocki1e4f63a2020-01-26 23:40:11 +01002529 policy->governor = new_gov;
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002530 ret = cpufreq_init_governor(policy);
Viresh Kumar4bc384a2015-07-18 11:31:03 +05302531 if (!ret) {
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002532 ret = cpufreq_start_governor(policy);
2533 if (!ret) {
Rafael J. Wysocki2bb40592019-02-20 00:25:18 +01002534 pr_debug("governor change\n");
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002535 return 0;
2536 }
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002537 cpufreq_exit_governor(policy);
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002538 }
2539
2540 /* new governor failed, so re-start old one */
2541 pr_debug("starting governor %s failed\n", policy->governor->name);
2542 if (old_gov) {
2543 policy->governor = old_gov;
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002544 if (cpufreq_init_governor(policy))
Viresh Kumar4bc384a2015-07-18 11:31:03 +05302545 policy->governor = NULL;
2546 else
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002547 cpufreq_start_governor(policy);
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002548 }
2549
Viresh Kumar4bc384a2015-07-18 11:31:03 +05302550 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002551}
Pavankumar Kondeti55839c62021-03-08 11:22:18 +05302552EXPORT_TRACEPOINT_SYMBOL_GPL(cpu_frequency_limits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002553
2554/**
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002555 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2556 * @cpu: CPU to re-evaluate the policy for.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557 *
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002558 * Update the current frequency for the cpufreq policy of @cpu and use
Viresh Kumar18c49922019-07-05 16:21:24 +05302559 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2560 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2561 * for the policy in question, among other things.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562 */
Rafael J. Wysocki30248fe2016-11-18 13:59:21 +01002563void cpufreq_update_policy(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002564{
Rafael J. Wysocki540a37582019-03-26 12:16:58 +01002565 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002566
Aaron Plattnerfefa8ff2014-06-18 11:27:32 -07002567 if (!policy)
Rafael J. Wysocki30248fe2016-11-18 13:59:21 +01002568 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002569
Viresh Kumarbb176f72013-06-19 14:19:33 +05302570 /*
2571 * BIOS might change freq behind our back
2572 * -> ask driver for current freq and notify governors about a change
2573 */
Viresh Kumar5ddc6d42019-06-20 08:35:48 +05302574 if (cpufreq_driver->get && has_target() &&
Viresh Kumar59807522019-06-20 08:35:49 +05302575 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
Rafael J. Wysocki348a2ec2019-02-20 00:24:25 +01002576 goto unlock;
Rafael J. Wysocki30248fe2016-11-18 13:59:21 +01002577
Viresh Kumar70a59fd2019-06-20 08:35:50 +05302578 refresh_frequency_limits(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002579
Aaron Plattnerfefa8ff2014-06-18 11:27:32 -07002580unlock:
Rafael J. Wysocki540a37582019-03-26 12:16:58 +01002581 cpufreq_cpu_release(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002582}
2583EXPORT_SYMBOL(cpufreq_update_policy);
2584
Rafael J. Wysocki5a25e3f2019-03-26 12:15:13 +01002585/**
2586 * cpufreq_update_limits - Update policy limits for a given CPU.
2587 * @cpu: CPU to update the policy limits for.
2588 *
2589 * Invoke the driver's ->update_limits callback if present or call
2590 * cpufreq_update_policy() for @cpu.
2591 */
2592void cpufreq_update_limits(unsigned int cpu)
2593{
2594 if (cpufreq_driver->update_limits)
2595 cpufreq_driver->update_limits(cpu);
2596 else
2597 cpufreq_update_policy(cpu);
2598}
2599EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2600
Linus Torvalds1da177e2005-04-16 15:20:36 -07002601/*********************************************************************
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002602 * BOOST *
2603 *********************************************************************/
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002604static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002605{
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002606 int ret;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002607
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002608 if (!policy->freq_table)
2609 return -ENXIO;
Rafael J. Wysocki552abb82020-05-18 12:49:45 +02002610
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002611 ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
2612 if (ret) {
2613 pr_err("%s: Policy frequency update failed\n", __func__);
2614 return ret;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002615 }
2616
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002617 ret = freq_qos_update_request(policy->max_freq_req, policy->max);
2618 if (ret < 0)
2619 return ret;
2620
Rafael J. Wysocki552abb82020-05-18 12:49:45 +02002621 return 0;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002622}
2623
2624int cpufreq_boost_trigger_state(int state)
2625{
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002626 struct cpufreq_policy *policy;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002627 unsigned long flags;
2628 int ret = 0;
2629
2630 if (cpufreq_driver->boost_enabled == state)
2631 return 0;
2632
2633 write_lock_irqsave(&cpufreq_driver_lock, flags);
2634 cpufreq_driver->boost_enabled = state;
2635 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2636
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002637 get_online_cpus();
2638 for_each_active_policy(policy) {
2639 ret = cpufreq_driver->set_boost(policy, state);
2640 if (ret)
2641 goto err_reset_state;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002642 }
Xiongfeng Wangcf6fada2020-05-30 10:08:30 +08002643 put_online_cpus();
2644
2645 return 0;
2646
2647err_reset_state:
2648 put_online_cpus();
2649
2650 write_lock_irqsave(&cpufreq_driver_lock, flags);
2651 cpufreq_driver->boost_enabled = !state;
2652 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2653
2654 pr_err("%s: Cannot %s BOOST\n",
2655 __func__, state ? "enable" : "disable");
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002656
2657 return ret;
2658}
2659
Rafael J. Wysocki41669da2015-12-27 00:23:48 +01002660static bool cpufreq_boost_supported(void)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002661{
Yue Hu89f98d72019-04-09 10:25:36 +08002662 return cpufreq_driver->set_boost;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002663}
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002664
Viresh Kumar44139ed2015-07-29 16:23:09 +05302665static int create_boost_sysfs_file(void)
2666{
2667 int ret;
2668
Viresh Kumarc82bd442015-10-15 21:35:23 +05302669 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
Viresh Kumar44139ed2015-07-29 16:23:09 +05302670 if (ret)
2671 pr_err("%s: cannot register global BOOST sysfs file\n",
2672 __func__);
2673
2674 return ret;
2675}
2676
2677static void remove_boost_sysfs_file(void)
2678{
2679 if (cpufreq_boost_supported())
Viresh Kumarc82bd442015-10-15 21:35:23 +05302680 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
Viresh Kumar44139ed2015-07-29 16:23:09 +05302681}
2682
2683int cpufreq_enable_boost_support(void)
2684{
2685 if (!cpufreq_driver)
2686 return -EINVAL;
2687
2688 if (cpufreq_boost_supported())
2689 return 0;
2690
Rafael J. Wysocki7a6c79f2015-12-27 00:27:38 +01002691 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
Viresh Kumar44139ed2015-07-29 16:23:09 +05302692
2693 /* This will get removed on driver unregister */
2694 return create_boost_sysfs_file();
2695}
2696EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2697
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002698int cpufreq_boost_enabled(void)
2699{
2700 return cpufreq_driver->boost_enabled;
2701}
2702EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2703
2704/*********************************************************************
Linus Torvalds1da177e2005-04-16 15:20:36 -07002705 * REGISTER / UNREGISTER CPUFREQ DRIVER *
2706 *********************************************************************/
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02002707static enum cpuhp_state hp_online;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002708
Chen Yuc4a3fa22017-04-09 13:45:16 +08002709static int cpuhp_cpufreq_online(unsigned int cpu)
2710{
2711 cpufreq_online(cpu);
2712
2713 return 0;
2714}
2715
2716static int cpuhp_cpufreq_offline(unsigned int cpu)
2717{
2718 cpufreq_offline(cpu);
2719
2720 return 0;
2721}
2722
Linus Torvalds1da177e2005-04-16 15:20:36 -07002723/**
2724 * cpufreq_register_driver - register a CPU Frequency driver
2725 * @driver_data: A struct cpufreq_driver containing the values#
2726 * submitted by the CPU Frequency driver.
2727 *
Viresh Kumarbb176f72013-06-19 14:19:33 +05302728 * Registers a CPU Frequency driver to this core code. This code
Eric Biggers63af4052016-02-20 21:50:01 -06002729 * returns zero on success, -EEXIST when another driver got here first
Dave Jones32ee8c32006-02-28 00:43:23 -05002730 * (and isn't unregistered in the meantime).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002731 *
2732 */
Linus Torvalds221dee22007-02-26 14:55:48 -08002733int cpufreq_register_driver(struct cpufreq_driver *driver_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002734{
2735 unsigned long flags;
2736 int ret;
2737
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002738 if (cpufreq_disabled())
2739 return -ENODEV;
2740
Viresh Kumar46770be2019-11-14 09:06:17 +05302741 /*
2742 * The cpufreq core depends heavily on the availability of device
2743 * structure, make sure they are available before proceeding further.
2744 */
2745 if (!get_cpu_device(0))
2746 return -EPROBE_DEFER;
2747
Linus Torvalds1da177e2005-04-16 15:20:36 -07002748 if (!driver_data || !driver_data->verify || !driver_data->init ||
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302749 !(driver_data->setpolicy || driver_data->target_index ||
Rafael J. Wysocki98322352014-03-19 12:48:30 +01002750 driver_data->target) ||
2751 (driver_data->setpolicy && (driver_data->target_index ||
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302752 driver_data->target)) ||
Viresh Kumara9a22b52019-02-14 16:16:21 +05302753 (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
Viresh Kumar91a12e92019-02-12 16:36:04 +05302754 (!driver_data->online != !driver_data->offline))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002755 return -EINVAL;
2756
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002757 pr_debug("trying to register driver %s\n", driver_data->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002758
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002759 /* Protect against concurrent CPU online/offline. */
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002760 cpus_read_lock();
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002761
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002762 write_lock_irqsave(&cpufreq_driver_lock, flags);
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002763 if (cpufreq_driver) {
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002764 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002765 ret = -EEXIST;
2766 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002767 }
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002768 cpufreq_driver = driver_data;
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002769 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002770
Ionela Voinescu874f6352020-09-01 21:55:47 +01002771 /*
2772 * Mark support for the scheduler's frequency invariance engine for
2773 * drivers that implement target(), target_index() or fast_switch().
2774 */
2775 if (!cpufreq_driver->setpolicy) {
2776 static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
2777 pr_debug("supports frequency invariance");
2778 }
2779
Viresh Kumarbc68b7d2015-01-02 12:34:30 +05302780 if (driver_data->setpolicy)
2781 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2782
Rafael J. Wysocki7a6c79f2015-12-27 00:27:38 +01002783 if (cpufreq_boost_supported()) {
2784 ret = create_boost_sysfs_file();
2785 if (ret)
2786 goto err_null_driver;
2787 }
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002788
Kay Sievers8a25a2f2011-12-21 14:29:42 -08002789 ret = subsys_interface_register(&cpufreq_interface);
Jiri Slaby8f5bc2a2011-03-01 17:41:10 +01002790 if (ret)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002791 goto err_boost_unreg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002792
Viresh Kumarce1bcfe2015-01-02 12:34:35 +05302793 if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2794 list_empty(&cpufreq_policy_list)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002795 /* if all ->init() calls failed, unregister */
David Arcari6c770032017-05-26 11:37:31 -04002796 ret = -ENODEV;
Viresh Kumarce1bcfe2015-01-02 12:34:35 +05302797 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2798 driver_data->name);
2799 goto err_if_unreg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002800 }
2801
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002802 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2803 "cpufreq:online",
2804 cpuhp_cpufreq_online,
2805 cpuhp_cpufreq_offline);
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02002806 if (ret < 0)
2807 goto err_if_unreg;
2808 hp_online = ret;
Sebastian Andrzej Siewior5372e052016-09-20 16:56:28 +02002809 ret = 0;
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02002810
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002811 pr_debug("driver %s up and running\n", driver_data->name);
Pankaj Gupta3834abb2016-05-16 11:07:19 +00002812 goto out;
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002813
Kay Sievers8a25a2f2011-12-21 14:29:42 -08002814err_if_unreg:
2815 subsys_interface_unregister(&cpufreq_interface);
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002816err_boost_unreg:
Viresh Kumar44139ed2015-07-29 16:23:09 +05302817 remove_boost_sysfs_file();
Jiri Slaby8f5bc2a2011-03-01 17:41:10 +01002818err_null_driver:
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002819 write_lock_irqsave(&cpufreq_driver_lock, flags);
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002820 cpufreq_driver = NULL;
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002821 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Pankaj Gupta3834abb2016-05-16 11:07:19 +00002822out:
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002823 cpus_read_unlock();
Pankaj Gupta3834abb2016-05-16 11:07:19 +00002824 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002825}
2826EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2827
Lee Jonesa9909c212020-07-15 09:26:23 +01002828/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002829 * cpufreq_unregister_driver - unregister the current CPUFreq driver
2830 *
Viresh Kumarbb176f72013-06-19 14:19:33 +05302831 * Unregister the current CPUFreq driver. Only call this if you have
Linus Torvalds1da177e2005-04-16 15:20:36 -07002832 * the right to do so, i.e. if you have succeeded in initialising before!
2833 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2834 * currently not initialised.
2835 */
Linus Torvalds221dee22007-02-26 14:55:48 -08002836int cpufreq_unregister_driver(struct cpufreq_driver *driver)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002837{
2838 unsigned long flags;
2839
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002840 if (!cpufreq_driver || (driver != cpufreq_driver))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002841 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002842
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002843 pr_debug("unregistering driver %s\n", driver->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002844
Sebastian Andrzej Siewior454d3a22015-07-22 17:59:11 +02002845 /* Protect against concurrent cpu hotplug */
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002846 cpus_read_lock();
Kay Sievers8a25a2f2011-12-21 14:29:42 -08002847 subsys_interface_unregister(&cpufreq_interface);
Viresh Kumar44139ed2015-07-29 16:23:09 +05302848 remove_boost_sysfs_file();
Ionela Voinescu874f6352020-09-01 21:55:47 +01002849 static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002850 cpuhp_remove_state_nocalls_cpuslocked(hp_online);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002852 write_lock_irqsave(&cpufreq_driver_lock, flags);
Viresh Kumar6eed9402013-08-06 22:53:11 +05302853
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002854 cpufreq_driver = NULL;
Viresh Kumar6eed9402013-08-06 22:53:11 +05302855
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002856 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002857 cpus_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002858
2859 return 0;
2860}
2861EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08002862
2863static int __init cpufreq_core_init(void)
2864{
Quentin Perret8412b452020-06-29 13:55:00 +05302865 struct cpufreq_governor *gov = cpufreq_default_governor();
2866
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002867 if (cpufreq_disabled())
2868 return -ENODEV;
2869
Viresh Kumar8eec1022015-10-15 21:35:22 +05302870 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
Thomas Renninger8aa84ad2009-07-24 15:25:05 +02002871 BUG_ON(!cpufreq_global_kobject);
2872
Quentin Perret8412b452020-06-29 13:55:00 +05302873 if (!strlen(default_governor))
2874 strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
2875
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08002876 return 0;
2877}
Len Brownd82f2692017-02-28 16:44:16 -05002878module_param(off, int, 0444);
Quentin Perret8412b452020-06-29 13:55:00 +05302879module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08002880core_initcall(cpufreq_core_init);