Merge "cpufreq: interactive: Replace per-cpu timer with per-policy timer" into msm-4.9
diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c
index 5c5f277..88b7f10 100644
--- a/drivers/cpufreq/cpufreq_interactive.c
+++ b/drivers/cpufreq/cpufreq_interactive.c
@@ -34,33 +34,36 @@
#define CREATE_TRACE_POINTS
#include <trace/events/cpufreq_interactive.h>
-struct cpufreq_interactive_cpuinfo {
- struct timer_list cpu_timer;
- struct timer_list cpu_slack_timer;
- spinlock_t load_lock; /* protects the next 4 fields */
- u64 time_in_idle;
- u64 time_in_idle_timestamp;
- u64 cputime_speedadj;
- u64 cputime_speedadj_timestamp;
+struct cpufreq_interactive_policyinfo {
+ struct timer_list policy_timer;
+ struct timer_list policy_slack_timer;
+ spinlock_t load_lock; /* protects load tracking stat */
u64 last_evaluated_jiffy;
struct cpufreq_policy *policy;
struct cpufreq_frequency_table *freq_table;
spinlock_t target_freq_lock; /*protects target freq */
unsigned int target_freq;
unsigned int floor_freq;
- u64 pol_floor_val_time; /* policy floor_validate_time */
- u64 loc_floor_val_time; /* per-cpu floor_validate_time */
- u64 pol_hispeed_val_time; /* policy hispeed_validate_time */
- u64 loc_hispeed_val_time; /* per-cpu hispeed_validate_time */
- u64 max_freq_hyst_start_time;
unsigned int min_freq;
+ u64 floor_validate_time;
+ u64 hispeed_validate_time;
+ u64 max_freq_hyst_start_time;
struct rw_semaphore enable_sem;
bool reject_notification;
int governor_enabled;
struct cpufreq_interactive_tunables *cached_tunables;
- int first_cpu;
};
+/* Protected by per-policy load_lock */
+struct cpufreq_interactive_cpuinfo {
+ u64 time_in_idle;
+ u64 time_in_idle_timestamp;
+ u64 cputime_speedadj;
+ u64 cputime_speedadj_timestamp;
+ unsigned int loadadjfreq;
+};
+
+static DEFINE_PER_CPU(struct cpufreq_interactive_policyinfo *, polinfo);
static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
/* realtime thread handles frequency scaling */
@@ -167,6 +170,7 @@ void cpufreq_put_global_kobject(void)
/* For cases where we have single governor instance for system */
static struct cpufreq_interactive_tunables *common_tunables;
+static struct cpufreq_interactive_tunables *cached_common_tunables;
static struct attribute_group *get_sysfs_attr(void);
@@ -197,64 +201,75 @@ static inline int set_window_helper(
static void cpufreq_interactive_timer_resched(unsigned long cpu,
bool slack_only)
{
- struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
+ struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, cpu);
+ struct cpufreq_interactive_cpuinfo *pcpu;
struct cpufreq_interactive_tunables *tunables =
- pcpu->policy->governor_data;
+ ppol->policy->governor_data;
u64 expires;
unsigned long flags;
+ int i;
- spin_lock_irqsave(&pcpu->load_lock, flags);
- expires = round_to_nw_start(pcpu->last_evaluated_jiffy, tunables);
+ spin_lock_irqsave(&ppol->load_lock, flags);
if (!slack_only) {
- pcpu->time_in_idle =
- get_cpu_idle_time(smp_processor_id(),
- &pcpu->time_in_idle_timestamp,
- tunables->io_is_busy);
- pcpu->cputime_speedadj = 0;
- pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
- del_timer(&pcpu->cpu_timer);
- pcpu->cpu_timer.expires = expires;
- add_timer_on(&pcpu->cpu_timer, cpu);
+ for_each_cpu(i, ppol->policy->cpus) {
+ pcpu = &per_cpu(cpuinfo, i);
+ pcpu->time_in_idle = get_cpu_idle_time(i,
+ &pcpu->time_in_idle_timestamp,
+ tunables->io_is_busy);
+ pcpu->cputime_speedadj = 0;
+ pcpu->cputime_speedadj_timestamp =
+ pcpu->time_in_idle_timestamp;
+ }
+ expires = round_to_nw_start(ppol->last_evaluated_jiffy,
+ tunables);
+ del_timer(&ppol->policy_timer);
+ ppol->policy_timer.expires = expires;
+ add_timer(&ppol->policy_timer);
}
if (tunables->timer_slack_val >= 0 &&
- pcpu->target_freq > pcpu->policy->min) {
+ ppol->target_freq > ppol->policy->min) {
expires += usecs_to_jiffies(tunables->timer_slack_val);
- del_timer(&pcpu->cpu_slack_timer);
- pcpu->cpu_slack_timer.expires = expires;
- add_timer_on(&pcpu->cpu_slack_timer, cpu);
+ del_timer(&ppol->policy_slack_timer);
+ ppol->policy_slack_timer.expires = expires;
+ add_timer(&ppol->policy_slack_timer);
}
- spin_unlock_irqrestore(&pcpu->load_lock, flags);
+ spin_unlock_irqrestore(&ppol->load_lock, flags);
}
/* The caller shall take enable_sem write semaphore to avoid any timer race.
- * The cpu_timer and cpu_slack_timer must be deactivated when calling this
- * function.
+ * The policy_timer and policy_slack_timer must be deactivated when calling
+ * this function.
*/
static void cpufreq_interactive_timer_start(
struct cpufreq_interactive_tunables *tunables, int cpu)
{
- struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
- u64 expires = round_to_nw_start(pcpu->last_evaluated_jiffy, tunables);
+ struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, cpu);
+ struct cpufreq_interactive_cpuinfo *pcpu;
+ u64 expires = round_to_nw_start(ppol->last_evaluated_jiffy, tunables);
unsigned long flags;
+ int i;
- spin_lock_irqsave(&pcpu->load_lock, flags);
- pcpu->cpu_timer.expires = expires;
- add_timer_on(&pcpu->cpu_timer, cpu);
+ spin_lock_irqsave(&ppol->load_lock, flags);
+ ppol->policy_timer.expires = expires;
+ add_timer(&ppol->policy_timer);
if (tunables->timer_slack_val >= 0 &&
- pcpu->target_freq > pcpu->policy->min) {
+ ppol->target_freq > ppol->policy->min) {
expires += usecs_to_jiffies(tunables->timer_slack_val);
- pcpu->cpu_slack_timer.expires = expires;
- add_timer_on(&pcpu->cpu_slack_timer, cpu);
+ ppol->policy_slack_timer.expires = expires;
+ add_timer(&ppol->policy_slack_timer);
}
- pcpu->time_in_idle =
- get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp,
- tunables->io_is_busy);
- pcpu->cputime_speedadj = 0;
- pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
- spin_unlock_irqrestore(&pcpu->load_lock, flags);
+ for_each_cpu(i, ppol->policy->cpus) {
+ pcpu = &per_cpu(cpuinfo, i);
+ pcpu->time_in_idle =
+ get_cpu_idle_time(i, &pcpu->time_in_idle_timestamp,
+ tunables->io_is_busy);
+ pcpu->cputime_speedadj = 0;
+ pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
+ }
+ spin_unlock_irqrestore(&ppol->load_lock, flags);
}
static unsigned int freq_to_above_hispeed_delay(
@@ -299,7 +314,7 @@ static unsigned int freq_to_targetload(
* choose_freq() will find the minimum frequency that does not exceed its
* target load given the current load.
*/
-static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pcpu,
+static unsigned int choose_freq(struct cpufreq_interactive_policyinfo *pcpu,
unsigned int loadadjfreq)
{
unsigned int freq = pcpu->policy->cur;
@@ -386,9 +401,10 @@ static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pcpu,
static u64 update_load(int cpu)
{
+ struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, cpu);
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
struct cpufreq_interactive_tunables *tunables =
- pcpu->policy->governor_data;
+ ppol->policy->governor_data;
u64 now;
u64 now_idle;
unsigned int delta_idle;
@@ -404,7 +420,7 @@ static u64 update_load(int cpu)
else
active_time = delta_time - delta_idle;
- pcpu->cputime_speedadj += active_time * pcpu->policy->cur;
+ pcpu->cputime_speedadj += active_time * ppol->policy->cur;
pcpu->time_in_idle = now_idle;
pcpu->time_in_idle_timestamp = now;
@@ -418,113 +434,119 @@ static void __cpufreq_interactive_timer(unsigned long data, bool is_notif)
unsigned int delta_time;
u64 cputime_speedadj;
int cpu_load;
- struct cpufreq_interactive_cpuinfo *pcpu =
- &per_cpu(cpuinfo, data);
+ struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, data);
struct cpufreq_interactive_tunables *tunables =
- pcpu->policy->governor_data;
+ ppol->policy->governor_data;
+ struct cpufreq_interactive_cpuinfo *pcpu;
unsigned int new_freq;
- unsigned int loadadjfreq;
+ unsigned int loadadjfreq = 0, tmploadadjfreq;
unsigned int index;
unsigned long flags;
- u64 max_fvtime;
- struct cpufreq_govinfo int_info;
+ unsigned long max_cpu;
+ int i;
+ struct cpufreq_govinfo govinfo;
- if (!down_read_trylock(&pcpu->enable_sem))
+ if (!down_read_trylock(&ppol->enable_sem))
return;
- if (!pcpu->governor_enabled)
+ if (!ppol->governor_enabled)
goto exit;
- spin_lock_irqsave(&pcpu->load_lock, flags);
- pcpu->last_evaluated_jiffy = get_jiffies_64();
- now = update_load(data);
- if (tunables->use_sched_load) {
- /*
- * Unlock early to avoid deadlock.
- *
- * load_change_callback() for thread migration already
- * holds rq lock. Then it locks load_lock to avoid racing
- * with cpufreq_interactive_timer_resched/start().
- * sched_get_busy() will also acquire rq lock. Thus we
- * can't hold load_lock when calling sched_get_busy().
- *
- * load_lock used in this function protects time
- * and load information. These stats are not used when
- * scheduler input is available. Thus unlocking load_lock
- * early is perfectly OK.
- */
- spin_unlock_irqrestore(&pcpu->load_lock, flags);
- cputime_speedadj = (u64)sched_get_busy(data) *
- pcpu->policy->cpuinfo.max_freq;
- do_div(cputime_speedadj, tunables->timer_rate);
- } else {
- delta_time = (unsigned int)
+ now = ktime_to_us(ktime_get());
+ spin_lock_irqsave(&ppol->load_lock, flags);
+ ppol->last_evaluated_jiffy = get_jiffies_64();
+
+ max_cpu = cpumask_first(ppol->policy->cpus);
+ for_each_cpu(i, ppol->policy->cpus) {
+ pcpu = &per_cpu(cpuinfo, i);
+ if (tunables->use_sched_load) {
+ cputime_speedadj = (u64)sched_get_busy(i) *
+ ppol->policy->cpuinfo.max_freq;
+ do_div(cputime_speedadj, tunables->timer_rate);
+ } else {
+ now = update_load(i);
+ delta_time = (unsigned int)
(now - pcpu->cputime_speedadj_timestamp);
- cputime_speedadj = pcpu->cputime_speedadj;
- spin_unlock_irqrestore(&pcpu->load_lock, flags);
- if (WARN_ON_ONCE(!delta_time))
- goto rearm;
- do_div(cputime_speedadj, delta_time);
+ if (WARN_ON_ONCE(!delta_time))
+ continue;
+ cputime_speedadj = pcpu->cputime_speedadj;
+ do_div(cputime_speedadj, delta_time);
+ }
+ tmploadadjfreq = (unsigned int)cputime_speedadj * 100;
+ pcpu->loadadjfreq = tmploadadjfreq;
+
+ if (tmploadadjfreq > loadadjfreq) {
+ loadadjfreq = tmploadadjfreq;
+ max_cpu = i;
+ }
+ }
+ spin_unlock_irqrestore(&ppol->load_lock, flags);
+
+ /*
+ * Send govinfo notification.
+ * Govinfo notification could potentially wake up another thread
+ * managed by its clients. Thread wakeups might trigger a load
+ * change callback that executes this function again. Therefore
+ * no spinlock could be held when sending the notification.
+ */
+ for_each_cpu(i, ppol->policy->cpus) {
+ pcpu = &per_cpu(cpuinfo, i);
+ govinfo.cpu = i;
+ govinfo.load = pcpu->loadadjfreq / ppol->policy->max;
+ govinfo.sampling_rate_us = tunables->timer_rate;
+ atomic_notifier_call_chain(&cpufreq_govinfo_notifier_list,
+ CPUFREQ_LOAD_CHANGE, &govinfo);
}
- loadadjfreq = (unsigned int)cputime_speedadj * 100;
-
- int_info.cpu = data;
- int_info.load = loadadjfreq / pcpu->policy->max;
- int_info.sampling_rate_us = tunables->timer_rate;
- atomic_notifier_call_chain(&cpufreq_govinfo_notifier_list,
- CPUFREQ_LOAD_CHANGE, &int_info);
-
- spin_lock_irqsave(&pcpu->target_freq_lock, flags);
- cpu_load = loadadjfreq / pcpu->policy->cur;
+ spin_lock_irqsave(&ppol->target_freq_lock, flags);
+ cpu_load = loadadjfreq / ppol->policy->cur;
tunables->boosted = tunables->boost_val || now < tunables->boostpulse_endtime;
if (cpu_load >= tunables->go_hispeed_load || tunables->boosted) {
- if (pcpu->policy->cur < tunables->hispeed_freq &&
+ if (ppol->policy->cur < tunables->hispeed_freq &&
cpu_load <= MAX_LOCAL_LOAD) {
new_freq = tunables->hispeed_freq;
} else {
- new_freq = choose_freq(pcpu, loadadjfreq);
+ new_freq = choose_freq(ppol, loadadjfreq);
if (new_freq < tunables->hispeed_freq)
new_freq = tunables->hispeed_freq;
}
} else {
- new_freq = choose_freq(pcpu, loadadjfreq);
+ new_freq = choose_freq(ppol, loadadjfreq);
if (new_freq > tunables->hispeed_freq &&
- pcpu->policy->cur < tunables->hispeed_freq)
+ ppol->policy->cur < tunables->hispeed_freq)
new_freq = tunables->hispeed_freq;
}
if (cpu_load <= MAX_LOCAL_LOAD &&
- pcpu->policy->cur >= tunables->hispeed_freq &&
- new_freq > pcpu->policy->cur &&
- now - pcpu->pol_hispeed_val_time <
- freq_to_above_hispeed_delay(tunables, pcpu->policy->cur)) {
+ ppol->policy->cur >= tunables->hispeed_freq &&
+ new_freq > ppol->policy->cur &&
+ now - ppol->hispeed_validate_time <
+ freq_to_above_hispeed_delay(tunables, ppol->policy->cur)) {
trace_cpufreq_interactive_notyet(
- data, cpu_load, pcpu->target_freq,
- pcpu->policy->cur, new_freq);
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
+ max_cpu, cpu_load, ppol->target_freq,
+ ppol->policy->cur, new_freq);
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
- pcpu->loc_hispeed_val_time = now;
+ ppol->hispeed_validate_time = now;
- if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
+ if (cpufreq_frequency_table_target(ppol->policy, ppol->freq_table,
new_freq, CPUFREQ_RELATION_L,
&index)) {
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
- new_freq = pcpu->freq_table[index].frequency;
+ new_freq = ppol->freq_table[index].frequency;
- if (!is_notif && pcpu->target_freq >= pcpu->policy->max
- && new_freq < pcpu->target_freq
- && now - pcpu->max_freq_hyst_start_time <
+ if (!is_notif && new_freq < ppol->target_freq &&
+ now - ppol->max_freq_hyst_start_time <
tunables->max_freq_hysteresis) {
- trace_cpufreq_interactive_notyet(data, cpu_load,
- pcpu->target_freq, pcpu->policy->cur, new_freq);
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
+ trace_cpufreq_interactive_notyet(max_cpu, cpu_load,
+ ppol->target_freq, ppol->policy->cur, new_freq);
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
@@ -532,14 +554,13 @@ static void __cpufreq_interactive_timer(unsigned long data, bool is_notif)
* Do not scale below floor_freq unless we have been at or above the
* floor frequency for the minimum sample time since last validated.
*/
- max_fvtime = max(pcpu->pol_floor_val_time, pcpu->loc_floor_val_time);
- if (!is_notif && new_freq < pcpu->floor_freq &&
- pcpu->target_freq >= pcpu->policy->cur) {
- if (now - max_fvtime < tunables->min_sample_time) {
+ if (!is_notif && new_freq < ppol->floor_freq) {
+ if (now - ppol->floor_validate_time <
+ tunables->min_sample_time) {
trace_cpufreq_interactive_notyet(
- data, cpu_load, pcpu->target_freq,
- pcpu->policy->cur, new_freq);
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
+ max_cpu, cpu_load, ppol->target_freq,
+ ppol->policy->cur, new_freq);
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
}
@@ -553,37 +574,38 @@ static void __cpufreq_interactive_timer(unsigned long data, bool is_notif)
*/
if (!tunables->boosted || new_freq > tunables->hispeed_freq) {
- pcpu->floor_freq = new_freq;
- if (pcpu->target_freq >= pcpu->policy->cur ||
- new_freq >= pcpu->policy->cur)
- pcpu->loc_floor_val_time = now;
+ ppol->floor_freq = new_freq;
+ ppol->floor_validate_time = now;
}
- if (pcpu->target_freq == new_freq &&
- pcpu->target_freq <= pcpu->policy->cur) {
+ if (new_freq == ppol->policy->max)
+ ppol->max_freq_hyst_start_time = now;
+
+ if (ppol->target_freq == new_freq &&
+ ppol->target_freq <= ppol->policy->cur) {
trace_cpufreq_interactive_already(
- data, cpu_load, pcpu->target_freq,
- pcpu->policy->cur, new_freq);
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
+ max_cpu, cpu_load, ppol->target_freq,
+ ppol->policy->cur, new_freq);
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
- trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
- pcpu->policy->cur, new_freq);
+ trace_cpufreq_interactive_target(max_cpu, cpu_load, ppol->target_freq,
+ ppol->policy->cur, new_freq);
- pcpu->target_freq = new_freq;
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
+ ppol->target_freq = new_freq;
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
spin_lock_irqsave(&speedchange_cpumask_lock, flags);
- cpumask_set_cpu(data, &speedchange_cpumask);
+ cpumask_set_cpu(max_cpu, &speedchange_cpumask);
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
wake_up_process(speedchange_task);
rearm:
- if (!timer_pending(&pcpu->cpu_timer))
+ if (!timer_pending(&ppol->policy_timer))
cpufreq_interactive_timer_resched(data, false);
exit:
- up_read(&pcpu->enable_sem);
+ up_read(&ppol->enable_sem);
return;
}
@@ -592,36 +614,12 @@ static void cpufreq_interactive_timer(unsigned long data)
__cpufreq_interactive_timer(data, false);
}
-static void cpufreq_interactive_idle_end(void)
-{
- struct cpufreq_interactive_cpuinfo *pcpu =
- &per_cpu(cpuinfo, smp_processor_id());
-
- if (!down_read_trylock(&pcpu->enable_sem))
- return;
- if (!pcpu->governor_enabled) {
- up_read(&pcpu->enable_sem);
- return;
- }
-
- /* Arm the timer for 1-2 ticks later if not already. */
- if (!timer_pending(&pcpu->cpu_timer)) {
- cpufreq_interactive_timer_resched(smp_processor_id(), false);
- } else if (time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
- del_timer(&pcpu->cpu_timer);
- del_timer(&pcpu->cpu_slack_timer);
- cpufreq_interactive_timer(smp_processor_id());
- }
-
- up_read(&pcpu->enable_sem);
-}
-
static int cpufreq_interactive_speedchange_task(void *data)
{
unsigned int cpu;
cpumask_t tmp_mask;
unsigned long flags;
- struct cpufreq_interactive_cpuinfo *pcpu;
+ struct cpufreq_interactive_policyinfo *ppol;
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
@@ -644,49 +642,22 @@ static int cpufreq_interactive_speedchange_task(void *data)
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
for_each_cpu(cpu, &tmp_mask) {
- unsigned int j;
- unsigned int max_freq = 0;
- struct cpufreq_interactive_cpuinfo *pjcpu;
- u64 hvt = ~0ULL, fvt = 0;
-
- pcpu = &per_cpu(cpuinfo, cpu);
- if (!down_read_trylock(&pcpu->enable_sem))
+ ppol = per_cpu(polinfo, cpu);
+ if (!down_read_trylock(&ppol->enable_sem))
continue;
- if (!pcpu->governor_enabled) {
- up_read(&pcpu->enable_sem);
+ if (!ppol->governor_enabled) {
+ up_read(&ppol->enable_sem);
continue;
}
- for_each_cpu(j, pcpu->policy->cpus) {
- pjcpu = &per_cpu(cpuinfo, j);
-
- fvt = max(fvt, pjcpu->loc_floor_val_time);
- if (pjcpu->target_freq > max_freq) {
- max_freq = pjcpu->target_freq;
- hvt = pjcpu->loc_hispeed_val_time;
- } else if (pjcpu->target_freq == max_freq) {
- hvt = min(hvt, pjcpu->loc_hispeed_val_time);
- }
- }
- for_each_cpu(j, pcpu->policy->cpus) {
- pjcpu = &per_cpu(cpuinfo, j);
- pjcpu->pol_floor_val_time = fvt;
- }
-
- if (max_freq != pcpu->policy->cur) {
- __cpufreq_driver_target(pcpu->policy,
- max_freq,
+ if (ppol->target_freq != ppol->policy->cur)
+ __cpufreq_driver_target(ppol->policy,
+ ppol->target_freq,
CPUFREQ_RELATION_H);
- for_each_cpu(j, pcpu->policy->cpus) {
- pjcpu = &per_cpu(cpuinfo, j);
- pjcpu->pol_hispeed_val_time = hvt;
- }
- }
trace_cpufreq_interactive_setspeed(cpu,
- pcpu->target_freq,
- pcpu->policy->cur);
-
- up_read(&pcpu->enable_sem);
+ ppol->target_freq,
+ ppol->policy->cur);
+ up_read(&ppol->enable_sem);
}
}
@@ -698,26 +669,35 @@ static void cpufreq_interactive_boost(struct cpufreq_interactive_tunables *tunab
int i;
int anyboost = 0;
unsigned long flags[2];
- struct cpufreq_interactive_cpuinfo *pcpu;
+ struct cpufreq_interactive_policyinfo *ppol;
tunables->boosted = true;
spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]);
for_each_online_cpu(i) {
- pcpu = &per_cpu(cpuinfo, i);
- if (tunables != pcpu->policy->governor_data)
+ ppol = per_cpu(polinfo, i);
+ if (!ppol || tunables != ppol->policy->governor_data)
continue;
- spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]);
- if (pcpu->target_freq < tunables->hispeed_freq) {
- pcpu->target_freq = tunables->hispeed_freq;
+ spin_lock_irqsave(&ppol->target_freq_lock, flags[1]);
+ if (ppol->target_freq < tunables->hispeed_freq) {
+ ppol->target_freq = tunables->hispeed_freq;
cpumask_set_cpu(i, &speedchange_cpumask);
- pcpu->pol_hispeed_val_time =
+ ppol->hispeed_validate_time =
ktime_to_us(ktime_get());
anyboost = 1;
}
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]);
+
+ /*
+ * Set floor freq and (re)start timer for when last
+ * validated.
+ */
+
+ ppol->floor_freq = tunables->hispeed_freq;
+ ppol->floor_validate_time = ktime_to_us(ktime_get());
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags[1]);
+ break;
}
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]);
@@ -730,33 +710,32 @@ static int load_change_callback(struct notifier_block *nb, unsigned long val,
void *data)
{
unsigned long cpu = (unsigned long) data;
- struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
+ struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, cpu);
struct cpufreq_interactive_tunables *tunables;
if (speedchange_task == current)
return 0;
-
- if (pcpu->reject_notification)
+ if (!ppol || ppol->reject_notification)
return 0;
- if (!down_read_trylock(&pcpu->enable_sem))
+ if (!down_read_trylock(&ppol->enable_sem))
return 0;
- if (!pcpu->governor_enabled) {
- up_read(&pcpu->enable_sem);
+ if (!ppol->governor_enabled) {
+ up_read(&ppol->enable_sem);
return 0;
}
- tunables = pcpu->policy->governor_data;
+ tunables = ppol->policy->governor_data;
if (!tunables->use_sched_load || !tunables->use_migration_notif) {
- up_read(&pcpu->enable_sem);
+ up_read(&ppol->enable_sem);
return 0;
}
trace_cpufreq_interactive_load_change(cpu);
- del_timer(&pcpu->cpu_timer);
- del_timer(&pcpu->cpu_slack_timer);
+ del_timer(&ppol->policy_timer);
+ del_timer(&ppol->policy_slack_timer);
__cpufreq_interactive_timer(cpu, true);
- up_read(&pcpu->enable_sem);
+ up_read(&ppol->enable_sem);
return 0;
}
@@ -768,38 +747,31 @@ static int cpufreq_interactive_notifier(
struct notifier_block *nb, unsigned long val, void *data)
{
struct cpufreq_freqs *freq = data;
- struct cpufreq_interactive_cpuinfo *pcpu;
+ struct cpufreq_interactive_policyinfo *ppol;
int cpu;
unsigned long flags;
if (val == CPUFREQ_POSTCHANGE) {
- pcpu = &per_cpu(cpuinfo, freq->cpu);
- if (!down_read_trylock(&pcpu->enable_sem))
+ ppol = per_cpu(polinfo, freq->cpu);
+ if (!ppol)
return 0;
- if (!pcpu->governor_enabled) {
- up_read(&pcpu->enable_sem);
+ if (!down_read_trylock(&ppol->enable_sem))
+ return 0;
+ if (!ppol->governor_enabled) {
+ up_read(&ppol->enable_sem);
return 0;
}
- for_each_cpu(cpu, pcpu->policy->cpus) {
- struct cpufreq_interactive_cpuinfo *pjcpu =
- &per_cpu(cpuinfo, cpu);
- if (cpu != freq->cpu) {
- if (!down_read_trylock(&pjcpu->enable_sem))
- continue;
- if (!pjcpu->governor_enabled) {
- up_read(&pjcpu->enable_sem);
- continue;
- }
- }
- spin_lock_irqsave(&pjcpu->load_lock, flags);
+ if (cpumask_first(ppol->policy->cpus) != freq->cpu) {
+ up_read(&ppol->enable_sem);
+ return 0;
+ }
+ spin_lock_irqsave(&ppol->load_lock, flags);
+ for_each_cpu(cpu, ppol->policy->cpus)
update_load(cpu);
- spin_unlock_irqrestore(&pjcpu->load_lock, flags);
- if (cpu != freq->cpu)
- up_read(&pjcpu->enable_sem);
- }
+ spin_unlock_irqrestore(&ppol->load_lock, flags);
- up_read(&pcpu->enable_sem);
+ up_read(&ppol->enable_sem);
}
return 0;
}
@@ -1041,7 +1013,9 @@ static ssize_t store_timer_rate(struct cpufreq_interactive_tunables *tunables,
return count;
for_each_possible_cpu(cpu) {
- t = per_cpu(cpuinfo, cpu).cached_tunables;
+ if (!per_cpu(polinfo, cpu))
+ continue;
+ t = per_cpu(polinfo, cpu)->cached_tunables;
if (t && t->use_sched_load)
t->timer_rate = val_round;
}
@@ -1161,7 +1135,9 @@ static ssize_t store_io_is_busy(struct cpufreq_interactive_tunables *tunables,
return count;
for_each_possible_cpu(cpu) {
- t = per_cpu(cpuinfo, cpu).cached_tunables;
+ if (!per_cpu(polinfo, cpu))
+ continue;
+ t = per_cpu(polinfo, cpu)->cached_tunables;
if (t && t->use_sched_load)
t->io_is_busy = val;
}
@@ -1181,7 +1157,9 @@ static int cpufreq_interactive_enable_sched_input(
set_window_count++;
if (set_window_count > 1) {
for_each_possible_cpu(j) {
- t = per_cpu(cpuinfo, j).cached_tunables;
+ if (!per_cpu(polinfo, j))
+ continue;
+ t = per_cpu(polinfo, j)->cached_tunables;
if (t && t->use_sched_load) {
tunables->timer_rate = t->timer_rate;
tunables->io_is_busy = t->io_is_busy;
@@ -1452,34 +1430,8 @@ static struct attribute_group *get_sysfs_attr(void)
return &interactive_attr_group_gov_sys;
}
-static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
- unsigned long val,
- void *data)
+static void cpufreq_interactive_nop_timer(unsigned long data)
{
- if (val == IDLE_END)
- cpufreq_interactive_idle_end();
-
- return 0;
-}
-
-static struct notifier_block cpufreq_interactive_idle_nb = {
- .notifier_call = cpufreq_interactive_idle_notifier,
-};
-
-static void save_tunables(struct cpufreq_policy *policy,
- struct cpufreq_interactive_tunables *tunables)
-{
- int cpu;
- struct cpufreq_interactive_cpuinfo *pcpu;
-
- if (have_governor_per_policy())
- cpu = cpumask_first(policy->related_cpus);
- else
- cpu = 0;
-
- pcpu = &per_cpu(cpuinfo, cpu);
- WARN_ON(pcpu->cached_tunables && pcpu->cached_tunables != tunables);
- pcpu->cached_tunables = tunables;
}
static struct cpufreq_interactive_tunables *alloc_tunable(
@@ -1505,29 +1457,67 @@ static struct cpufreq_interactive_tunables *alloc_tunable(
spin_lock_init(&tunables->target_loads_lock);
spin_lock_init(&tunables->above_hispeed_delay_lock);
- save_tunables(policy, tunables);
return tunables;
}
-static struct cpufreq_interactive_tunables *restore_tunables(
- struct cpufreq_policy *policy)
+static struct cpufreq_interactive_policyinfo *get_policyinfo(
+ struct cpufreq_policy *policy)
{
- int cpu;
+ struct cpufreq_interactive_policyinfo *ppol =
+ per_cpu(polinfo, policy->cpu);
+ int i;
+ /* polinfo already allocated for policy, return */
+ if (ppol)
+ return ppol;
+
+ ppol = kzalloc(sizeof(*ppol), GFP_KERNEL);
+ if (!ppol)
+ return ERR_PTR(-ENOMEM);
+
+ init_timer_deferrable(&ppol->policy_timer);
+ ppol->policy_timer.function = cpufreq_interactive_timer;
+ init_timer(&ppol->policy_slack_timer);
+ ppol->policy_slack_timer.function = cpufreq_interactive_nop_timer;
+ spin_lock_init(&ppol->load_lock);
+ spin_lock_init(&ppol->target_freq_lock);
+ init_rwsem(&ppol->enable_sem);
+
+ for_each_cpu(i, policy->related_cpus)
+ per_cpu(polinfo, i) = ppol;
+ return ppol;
+}
+
+/* This function is not multithread-safe. */
+static void free_policyinfo(int cpu)
+{
+ struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, cpu);
+ int j;
+
+ if (!ppol)
+ return;
+
+ for_each_possible_cpu(j)
+ if (per_cpu(polinfo, j) == ppol)
+ per_cpu(polinfo, cpu) = NULL;
+ kfree(ppol->cached_tunables);
+ kfree(ppol);
+}
+
+static struct cpufreq_interactive_tunables *get_tunables(
+ struct cpufreq_interactive_policyinfo *ppol)
+{
if (have_governor_per_policy())
- cpu = cpumask_first(policy->related_cpus);
+ return ppol->cached_tunables;
else
- cpu = 0;
-
- return per_cpu(cpuinfo, cpu).cached_tunables;
+ return cached_common_tunables;
}
static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
unsigned int event)
{
int rc;
- unsigned int j;
- struct cpufreq_interactive_cpuinfo *pcpu;
+ struct cpufreq_interactive_policyinfo *ppol;
struct cpufreq_frequency_table *freq_table;
struct cpufreq_interactive_tunables *tunables;
unsigned long flags;
@@ -1541,6 +1531,10 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
switch (event) {
case CPUFREQ_GOV_POLICY_INIT:
+ ppol = get_policyinfo(policy);
+ if (IS_ERR(ppol))
+ return PTR_ERR(ppol);
+
if (have_governor_per_policy()) {
WARN_ON(tunables);
} else if (tunables) {
@@ -1549,7 +1543,7 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
return 0;
}
- tunables = restore_tunables(policy);
+ tunables = get_tunables(ppol);
if (!tunables) {
tunables = alloc_tunable(policy);
if (IS_ERR(tunables))
@@ -1571,24 +1565,25 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
return rc;
}
- if (!policy->governor->initialized) {
- idle_notifier_register(&cpufreq_interactive_idle_nb);
+ if (!policy->governor->initialized)
cpufreq_register_notifier(&cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
- }
if (tunables->use_sched_load)
cpufreq_interactive_enable_sched_input(tunables);
+ if (have_governor_per_policy())
+ ppol->cached_tunables = tunables;
+ else
+ cached_common_tunables = tunables;
+
break;
case CPUFREQ_GOV_POLICY_EXIT:
if (!--tunables->usage_count) {
- if (policy->governor->initialized == 1) {
+ if (policy->governor->initialized == 1)
cpufreq_unregister_notifier(&cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
- idle_notifier_unregister(&cpufreq_interactive_idle_nb);
- }
sysfs_remove_group(get_governor_parent_kobj(policy),
get_sysfs_attr());
@@ -1610,44 +1605,40 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
if (!tunables->hispeed_freq)
tunables->hispeed_freq = policy->max;
- for_each_cpu(j, policy->cpus) {
- pcpu = &per_cpu(cpuinfo, j);
- pcpu->policy = policy;
- pcpu->target_freq = policy->cur;
- pcpu->freq_table = freq_table;
- pcpu->floor_freq = pcpu->target_freq;
- pcpu->pol_floor_val_time =
- ktime_to_us(ktime_get());
- pcpu->loc_floor_val_time = pcpu->pol_floor_val_time;
- pcpu->pol_hispeed_val_time = pcpu->pol_floor_val_time;
- pcpu->loc_hispeed_val_time = pcpu->pol_floor_val_time;
- pcpu->min_freq = policy->min;
- pcpu->reject_notification = true;
- down_write(&pcpu->enable_sem);
- del_timer_sync(&pcpu->cpu_timer);
- del_timer_sync(&pcpu->cpu_slack_timer);
- pcpu->last_evaluated_jiffy = get_jiffies_64();
- cpufreq_interactive_timer_start(tunables, j);
- pcpu->governor_enabled = 1;
- up_write(&pcpu->enable_sem);
- pcpu->reject_notification = false;
- }
+ ppol = per_cpu(polinfo, policy->cpu);
+ ppol->policy = policy;
+ ppol->target_freq = policy->cur;
+ ppol->freq_table = freq_table;
+ ppol->floor_freq = ppol->target_freq;
+ ppol->floor_validate_time = ktime_to_us(ktime_get());
+ ppol->hispeed_validate_time = ppol->floor_validate_time;
+ ppol->min_freq = policy->min;
+ ppol->reject_notification = true;
+ down_write(&ppol->enable_sem);
+ del_timer_sync(&ppol->policy_timer);
+ del_timer_sync(&ppol->policy_slack_timer);
+ ppol->policy_timer.data = policy->cpu;
+ ppol->last_evaluated_jiffy = get_jiffies_64();
+ cpufreq_interactive_timer_start(tunables, policy->cpu);
+ ppol->governor_enabled = 1;
+ up_write(&ppol->enable_sem);
+ ppol->reject_notification = false;
mutex_unlock(&gov_lock);
break;
case CPUFREQ_GOV_STOP:
mutex_lock(&gov_lock);
- for_each_cpu(j, policy->cpus) {
- pcpu = &per_cpu(cpuinfo, j);
- pcpu->reject_notification = true;
- down_write(&pcpu->enable_sem);
- pcpu->governor_enabled = 0;
- del_timer_sync(&pcpu->cpu_timer);
- del_timer_sync(&pcpu->cpu_slack_timer);
- up_write(&pcpu->enable_sem);
- pcpu->reject_notification = false;
- }
+
+ ppol = per_cpu(polinfo, policy->cpu);
+ ppol->reject_notification = true;
+ down_write(&ppol->enable_sem);
+ ppol->governor_enabled = 0;
+ ppol->target_freq = 0;
+ del_timer_sync(&ppol->policy_timer);
+ del_timer_sync(&ppol->policy_slack_timer);
+ up_write(&ppol->enable_sem);
+ ppol->reject_notification = false;
mutex_unlock(&gov_lock);
break;
@@ -1655,29 +1646,25 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
case CPUFREQ_GOV_LIMITS:
__cpufreq_driver_target(policy,
policy->cur, CPUFREQ_RELATION_L);
- for_each_cpu(j, policy->cpus) {
- pcpu = &per_cpu(cpuinfo, j);
- down_read(&pcpu->enable_sem);
- if (pcpu->governor_enabled == 0) {
- up_read(&pcpu->enable_sem);
- continue;
- }
+ ppol = per_cpu(polinfo, policy->cpu);
- spin_lock_irqsave(&pcpu->target_freq_lock, flags);
- if (policy->max < pcpu->target_freq)
- pcpu->target_freq = policy->max;
- else if (policy->min > pcpu->target_freq)
- pcpu->target_freq = policy->min;
+ down_read(&ppol->enable_sem);
+ if (ppol->governor_enabled) {
+ spin_lock_irqsave(&ppol->target_freq_lock, flags);
+ if (policy->max < ppol->target_freq)
+ ppol->target_freq = policy->max;
+ else if (policy->min > ppol->target_freq)
+ ppol->target_freq = policy->min;
+ spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
- spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
-
- if (policy->min < pcpu->min_freq)
- cpufreq_interactive_timer_resched(j, true);
- pcpu->min_freq = policy->min;
-
- up_read(&pcpu->enable_sem);
+ if (policy->min < ppol->min_freq)
+ cpufreq_interactive_timer_resched(policy->cpu,
+ true);
+ ppol->min_freq = policy->min;
}
+ up_read(&ppol->enable_sem);
+
break;
}
return 0;
@@ -1693,29 +1680,10 @@ struct cpufreq_governor cpufreq_gov_interactive = {
.owner = THIS_MODULE,
};
-static void cpufreq_interactive_nop_timer(unsigned long data)
-{
-}
-
static int __init cpufreq_interactive_init(void)
{
- unsigned int i;
- struct cpufreq_interactive_cpuinfo *pcpu;
struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
- /* Initalize per-cpu timers */
- for_each_possible_cpu(i) {
- pcpu = &per_cpu(cpuinfo, i);
- init_timer_deferrable(&pcpu->cpu_timer);
- pcpu->cpu_timer.function = cpufreq_interactive_timer;
- pcpu->cpu_timer.data = i;
- init_timer(&pcpu->cpu_slack_timer);
- pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer;
- spin_lock_init(&pcpu->load_lock);
- spin_lock_init(&pcpu->target_freq_lock);
- init_rwsem(&pcpu->enable_sem);
- }
-
spin_lock_init(&speedchange_cpumask_lock);
mutex_init(&gov_lock);
mutex_init(&sched_lock);
@@ -1743,17 +1711,13 @@ module_init(cpufreq_interactive_init);
static void __exit cpufreq_interactive_exit(void)
{
int cpu;
- struct cpufreq_interactive_cpuinfo *pcpu;
cpufreq_unregister_governor(&cpufreq_gov_interactive);
kthread_stop(speedchange_task);
put_task_struct(speedchange_task);
- for_each_possible_cpu(cpu) {
- pcpu = &per_cpu(cpuinfo, cpu);
- kfree(pcpu->cached_tunables);
- pcpu->cached_tunables = NULL;
- }
+ for_each_possible_cpu(cpu)
+ free_policyinfo(cpu);
}
module_exit(cpufreq_interactive_exit);