blob: ecea8d9f957c1eaf3aa9bc9a9103207280de021a [file] [log] [blame]
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02001/*
2 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
3 *
4 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
5 *
6 * Interactivity improvements by Mike Galbraith
7 * (C) 2007 Mike Galbraith <efault@gmx.de>
8 *
9 * Various enhancements by Dmitry Adamushko.
10 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
11 *
12 * Group scheduling enhancements by Srivatsa Vaddagiri
13 * Copyright IBM Corporation, 2007
14 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
15 *
16 * Scaled math optimizations by Thomas Gleixner
17 * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
Peter Zijlstra21805082007-08-25 18:41:53 +020018 *
19 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra
20 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020021 */
22
Arjan van de Ven97455122008-01-25 21:08:34 +010023#include <linux/latencytop.h>
Christian Ehrhardt1983a922009-11-30 12:16:47 +010024#include <linux/sched.h>
Sisir Koppaka3436ae12011-03-26 18:22:55 +053025#include <linux/cpumask.h>
Peter Zijlstra029632f2011-10-25 10:00:11 +020026#include <linux/slab.h>
27#include <linux/profile.h>
28#include <linux/interrupt.h>
Peter Zijlstracbee9f82012-10-25 14:16:43 +020029#include <linux/mempolicy.h>
Mel Gormane14808b2012-11-19 10:59:15 +000030#include <linux/migrate.h>
Peter Zijlstracbee9f82012-10-25 14:16:43 +020031#include <linux/task_work.h>
Peter Zijlstra029632f2011-10-25 10:00:11 +020032
33#include <trace/events/sched.h>
34
35#include "sched.h"
Arjan van de Ven97455122008-01-25 21:08:34 +010036
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020037/*
Peter Zijlstra21805082007-08-25 18:41:53 +020038 * Targeted preemption latency for CPU-bound tasks:
Takuya Yoshikawa864616e2010-10-14 16:09:13 +090039 * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020040 *
Peter Zijlstra21805082007-08-25 18:41:53 +020041 * NOTE: this latency value is not the same as the concept of
Ingo Molnard274a4c2007-10-15 17:00:14 +020042 * 'timeslice length' - timeslices in CFS are of variable length
43 * and have no persistent notion like in traditional, time-slice
44 * based scheduling concepts.
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020045 *
Ingo Molnard274a4c2007-10-15 17:00:14 +020046 * (to see the precise effective timeslice length of your workload,
47 * run vmstat and monitor the context-switches (cs) field)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020048 */
Mike Galbraith21406922010-03-11 17:17:15 +010049unsigned int sysctl_sched_latency = 6000000ULL;
50unsigned int normalized_sysctl_sched_latency = 6000000ULL;
Ingo Molnar2bd8e6d2007-10-15 17:00:02 +020051
52/*
Christian Ehrhardt1983a922009-11-30 12:16:47 +010053 * The initial- and re-scaling of tunables is configurable
54 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
55 *
56 * Options are:
57 * SCHED_TUNABLESCALING_NONE - unscaled, always *1
58 * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
59 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
60 */
61enum sched_tunable_scaling sysctl_sched_tunable_scaling
62 = SCHED_TUNABLESCALING_LOG;
63
64/*
Peter Zijlstrab2be5e92007-11-09 22:39:37 +010065 * Minimal preemption granularity for CPU-bound tasks:
Takuya Yoshikawa864616e2010-10-14 16:09:13 +090066 * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
Peter Zijlstrab2be5e92007-11-09 22:39:37 +010067 */
Ingo Molnar0bf377b2010-09-12 08:14:52 +020068unsigned int sysctl_sched_min_granularity = 750000ULL;
69unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;
Peter Zijlstrab2be5e92007-11-09 22:39:37 +010070
71/*
72 * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
73 */
Ingo Molnar0bf377b2010-09-12 08:14:52 +020074static unsigned int sched_nr_latency = 8;
Peter Zijlstrab2be5e92007-11-09 22:39:37 +010075
76/*
Mike Galbraith2bba22c2009-09-09 15:41:37 +020077 * After fork, child runs first. If set to 0 (default) then
Ingo Molnar2bd8e6d2007-10-15 17:00:02 +020078 * parent will (try to) run first.
79 */
Mike Galbraith2bba22c2009-09-09 15:41:37 +020080unsigned int sysctl_sched_child_runs_first __read_mostly;
Peter Zijlstra21805082007-08-25 18:41:53 +020081
82/*
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020083 * SCHED_OTHER wake-up granularity.
Mike Galbraith172e0822009-09-09 15:41:37 +020084 * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020085 *
86 * This option delays the preemption effects of decoupled workloads
87 * and reduces their over-scheduling. Synchronous workloads will still
88 * have immediate wakeup/sleep latencies.
89 */
Mike Galbraith172e0822009-09-09 15:41:37 +020090unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +010091unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +020092
Ingo Molnarda84d962007-10-15 17:00:18 +020093const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
94
Paul Turnera7a4f8a2010-11-15 15:47:06 -080095/*
96 * The exponential sliding window over which load is averaged for shares
97 * distribution.
98 * (default: 10msec)
99 */
100unsigned int __read_mostly sysctl_sched_shares_window = 10000000UL;
101
Paul Turnerec12cb72011-07-21 09:43:30 -0700102#ifdef CONFIG_CFS_BANDWIDTH
103/*
104 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
105 * each time a cfs_rq requests quota.
106 *
107 * Note: in the case that the slice exceeds the runtime remaining (either due
108 * to consumption or the quota being specified to be smaller than the slice)
109 * we will always only issue the remaining available time.
110 *
111 * default: 5 msec, units: microseconds
112 */
113unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
114#endif
115
Paul Gortmaker85276322013-04-19 15:10:50 -0400116static inline void update_load_add(struct load_weight *lw, unsigned long inc)
117{
118 lw->weight += inc;
119 lw->inv_weight = 0;
120}
121
122static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
123{
124 lw->weight -= dec;
125 lw->inv_weight = 0;
126}
127
128static inline void update_load_set(struct load_weight *lw, unsigned long w)
129{
130 lw->weight = w;
131 lw->inv_weight = 0;
132}
133
Peter Zijlstra029632f2011-10-25 10:00:11 +0200134/*
135 * Increase the granularity value when there are more CPUs,
136 * because with more CPUs the 'effective latency' as visible
137 * to users decreases. But the relationship is not linear,
138 * so pick a second-best guess by going with the log2 of the
139 * number of CPUs.
140 *
141 * This idea comes from the SD scheduler of Con Kolivas:
142 */
143static int get_update_sysctl_factor(void)
144{
145 unsigned int cpus = min_t(int, num_online_cpus(), 8);
146 unsigned int factor;
147
148 switch (sysctl_sched_tunable_scaling) {
149 case SCHED_TUNABLESCALING_NONE:
150 factor = 1;
151 break;
152 case SCHED_TUNABLESCALING_LINEAR:
153 factor = cpus;
154 break;
155 case SCHED_TUNABLESCALING_LOG:
156 default:
157 factor = 1 + ilog2(cpus);
158 break;
159 }
160
161 return factor;
162}
163
164static void update_sysctl(void)
165{
166 unsigned int factor = get_update_sysctl_factor();
167
168#define SET_SYSCTL(name) \
169 (sysctl_##name = (factor) * normalized_sysctl_##name)
170 SET_SYSCTL(sched_min_granularity);
171 SET_SYSCTL(sched_latency);
172 SET_SYSCTL(sched_wakeup_granularity);
173#undef SET_SYSCTL
174}
175
176void sched_init_granularity(void)
177{
178 update_sysctl();
179}
180
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100181#define WMULT_CONST (~0U)
Peter Zijlstra029632f2011-10-25 10:00:11 +0200182#define WMULT_SHIFT 32
183
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100184static void __update_inv_weight(struct load_weight *lw)
Peter Zijlstra029632f2011-10-25 10:00:11 +0200185{
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100186 unsigned long w;
Peter Zijlstra029632f2011-10-25 10:00:11 +0200187
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100188 if (likely(lw->inv_weight))
189 return;
190
191 w = scale_load_down(lw->weight);
192
193 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
194 lw->inv_weight = 1;
195 else if (unlikely(!w))
196 lw->inv_weight = WMULT_CONST;
Peter Zijlstra029632f2011-10-25 10:00:11 +0200197 else
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100198 lw->inv_weight = WMULT_CONST / w;
199}
Peter Zijlstra029632f2011-10-25 10:00:11 +0200200
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100201/*
202 * delta_exec * weight / lw.weight
203 * OR
204 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
205 *
206 * Either weight := NICE_0_LOAD and lw \e prio_to_wmult[], in which case
207 * we're guaranteed shift stays positive because inv_weight is guaranteed to
208 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
209 *
210 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
211 * weight/lw.weight <= 1, and therefore our shift will also be positive.
212 */
213static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
214{
215 u64 fact = scale_load_down(weight);
216 int shift = WMULT_SHIFT;
Peter Zijlstra029632f2011-10-25 10:00:11 +0200217
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100218 __update_inv_weight(lw);
219
220 if (unlikely(fact >> 32)) {
221 while (fact >> 32) {
222 fact >>= 1;
223 shift--;
224 }
Peter Zijlstra029632f2011-10-25 10:00:11 +0200225 }
226
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100227 /* hint to use a 32x32->64 mul */
228 fact = (u64)(u32)fact * lw->inv_weight;
Peter Zijlstra029632f2011-10-25 10:00:11 +0200229
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100230 while (fact >> 32) {
231 fact >>= 1;
232 shift--;
233 }
234
235 return mul_u64_u32_shr(delta_exec, fact, shift);
Peter Zijlstra029632f2011-10-25 10:00:11 +0200236}
237
238
239const struct sched_class fair_sched_class;
Peter Zijlstraa4c2f002008-10-17 19:27:03 +0200240
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200241/**************************************************************
242 * CFS operations on generic schedulable entities:
243 */
244
245#ifdef CONFIG_FAIR_GROUP_SCHED
246
247/* cpu runqueue to which this cfs_rq is attached */
248static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
249{
250 return cfs_rq->rq;
251}
252
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200253/* An entity is a task if it doesn't "own" a runqueue */
254#define entity_is_task(se) (!se->my_q)
255
Peter Zijlstra8f488942009-07-24 12:25:30 +0200256static inline struct task_struct *task_of(struct sched_entity *se)
257{
258#ifdef CONFIG_SCHED_DEBUG
259 WARN_ON_ONCE(!entity_is_task(se));
260#endif
261 return container_of(se, struct task_struct, se);
262}
263
Peter Zijlstrab7581492008-04-19 19:45:00 +0200264/* Walk up scheduling entities hierarchy */
265#define for_each_sched_entity(se) \
266 for (; se; se = se->parent)
267
268static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
269{
270 return p->se.cfs_rq;
271}
272
273/* runqueue on which this entity is (to be) queued */
274static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
275{
276 return se->cfs_rq;
277}
278
279/* runqueue "owned" by this group */
280static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
281{
282 return grp->my_q;
283}
284
Paul Turneraff3e492012-10-04 13:18:30 +0200285static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq,
286 int force_update);
Paul Turner9ee474f2012-10-04 13:18:30 +0200287
Peter Zijlstra3d4b47b2010-11-15 15:47:01 -0800288static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
289{
290 if (!cfs_rq->on_list) {
Paul Turner67e86252010-11-15 15:47:05 -0800291 /*
292 * Ensure we either appear before our parent (if already
293 * enqueued) or force our parent to appear after us when it is
294 * enqueued. The fact that we always enqueue bottom-up
295 * reduces this to two cases.
296 */
297 if (cfs_rq->tg->parent &&
298 cfs_rq->tg->parent->cfs_rq[cpu_of(rq_of(cfs_rq))]->on_list) {
299 list_add_rcu(&cfs_rq->leaf_cfs_rq_list,
Peter Zijlstra3d4b47b2010-11-15 15:47:01 -0800300 &rq_of(cfs_rq)->leaf_cfs_rq_list);
Paul Turner67e86252010-11-15 15:47:05 -0800301 } else {
302 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
303 &rq_of(cfs_rq)->leaf_cfs_rq_list);
304 }
Peter Zijlstra3d4b47b2010-11-15 15:47:01 -0800305
306 cfs_rq->on_list = 1;
Paul Turner9ee474f2012-10-04 13:18:30 +0200307 /* We should have no load, but we need to update last_decay. */
Paul Turneraff3e492012-10-04 13:18:30 +0200308 update_cfs_rq_blocked_load(cfs_rq, 0);
Peter Zijlstra3d4b47b2010-11-15 15:47:01 -0800309 }
310}
311
312static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
313{
314 if (cfs_rq->on_list) {
315 list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
316 cfs_rq->on_list = 0;
317 }
318}
319
Peter Zijlstrab7581492008-04-19 19:45:00 +0200320/* Iterate thr' all leaf cfs_rq's on a runqueue */
321#define for_each_leaf_cfs_rq(rq, cfs_rq) \
322 list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list)
323
324/* Do the two (enqueued) entities belong to the same group ? */
Peter Zijlstrafed14d42012-02-11 06:05:00 +0100325static inline struct cfs_rq *
Peter Zijlstrab7581492008-04-19 19:45:00 +0200326is_same_group(struct sched_entity *se, struct sched_entity *pse)
327{
328 if (se->cfs_rq == pse->cfs_rq)
Peter Zijlstrafed14d42012-02-11 06:05:00 +0100329 return se->cfs_rq;
Peter Zijlstrab7581492008-04-19 19:45:00 +0200330
Peter Zijlstrafed14d42012-02-11 06:05:00 +0100331 return NULL;
Peter Zijlstrab7581492008-04-19 19:45:00 +0200332}
333
334static inline struct sched_entity *parent_entity(struct sched_entity *se)
335{
336 return se->parent;
337}
338
Peter Zijlstra464b7522008-10-24 11:06:15 +0200339static void
340find_matching_se(struct sched_entity **se, struct sched_entity **pse)
341{
342 int se_depth, pse_depth;
343
344 /*
345 * preemption test can be made between sibling entities who are in the
346 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
347 * both tasks until we find their ancestors who are siblings of common
348 * parent.
349 */
350
351 /* First walk up until both entities are at same depth */
Peter Zijlstrafed14d42012-02-11 06:05:00 +0100352 se_depth = (*se)->depth;
353 pse_depth = (*pse)->depth;
Peter Zijlstra464b7522008-10-24 11:06:15 +0200354
355 while (se_depth > pse_depth) {
356 se_depth--;
357 *se = parent_entity(*se);
358 }
359
360 while (pse_depth > se_depth) {
361 pse_depth--;
362 *pse = parent_entity(*pse);
363 }
364
365 while (!is_same_group(*se, *pse)) {
366 *se = parent_entity(*se);
367 *pse = parent_entity(*pse);
368 }
369}
370
Peter Zijlstra8f488942009-07-24 12:25:30 +0200371#else /* !CONFIG_FAIR_GROUP_SCHED */
372
373static inline struct task_struct *task_of(struct sched_entity *se)
374{
375 return container_of(se, struct task_struct, se);
376}
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200377
378static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
379{
380 return container_of(cfs_rq, struct rq, cfs);
381}
382
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200383#define entity_is_task(se) 1
384
Peter Zijlstrab7581492008-04-19 19:45:00 +0200385#define for_each_sched_entity(se) \
386 for (; se; se = NULL)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200387
Peter Zijlstrab7581492008-04-19 19:45:00 +0200388static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200389{
Peter Zijlstrab7581492008-04-19 19:45:00 +0200390 return &task_rq(p)->cfs;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200391}
392
Peter Zijlstrab7581492008-04-19 19:45:00 +0200393static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
394{
395 struct task_struct *p = task_of(se);
396 struct rq *rq = task_rq(p);
397
398 return &rq->cfs;
399}
400
401/* runqueue "owned" by this group */
402static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
403{
404 return NULL;
405}
406
Peter Zijlstra3d4b47b2010-11-15 15:47:01 -0800407static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
408{
409}
410
411static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
412{
413}
414
Peter Zijlstrab7581492008-04-19 19:45:00 +0200415#define for_each_leaf_cfs_rq(rq, cfs_rq) \
416 for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL)
417
Peter Zijlstrab7581492008-04-19 19:45:00 +0200418static inline struct sched_entity *parent_entity(struct sched_entity *se)
419{
420 return NULL;
421}
422
Peter Zijlstra464b7522008-10-24 11:06:15 +0200423static inline void
424find_matching_se(struct sched_entity **se, struct sched_entity **pse)
425{
426}
427
Peter Zijlstrab7581492008-04-19 19:45:00 +0200428#endif /* CONFIG_FAIR_GROUP_SCHED */
429
Peter Zijlstra6c16a6d2012-03-21 13:07:16 -0700430static __always_inline
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100431void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200432
433/**************************************************************
434 * Scheduling class tree data structure manipulation methods:
435 */
436
Andrei Epure1bf08232013-03-12 21:12:24 +0200437static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
Peter Zijlstra02e04312007-10-15 17:00:07 +0200438{
Andrei Epure1bf08232013-03-12 21:12:24 +0200439 s64 delta = (s64)(vruntime - max_vruntime);
Peter Zijlstra368059a2007-10-15 17:00:11 +0200440 if (delta > 0)
Andrei Epure1bf08232013-03-12 21:12:24 +0200441 max_vruntime = vruntime;
Peter Zijlstra02e04312007-10-15 17:00:07 +0200442
Andrei Epure1bf08232013-03-12 21:12:24 +0200443 return max_vruntime;
Peter Zijlstra02e04312007-10-15 17:00:07 +0200444}
445
Ingo Molnar0702e3e2007-10-15 17:00:14 +0200446static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
Peter Zijlstrab0ffd242007-10-15 17:00:12 +0200447{
448 s64 delta = (s64)(vruntime - min_vruntime);
449 if (delta < 0)
450 min_vruntime = vruntime;
451
452 return min_vruntime;
453}
454
Fabio Checconi54fdc582009-07-16 12:32:27 +0200455static inline int entity_before(struct sched_entity *a,
456 struct sched_entity *b)
457{
458 return (s64)(a->vruntime - b->vruntime) < 0;
459}
460
Peter Zijlstra1af5f732008-10-24 11:06:13 +0200461static void update_min_vruntime(struct cfs_rq *cfs_rq)
462{
463 u64 vruntime = cfs_rq->min_vruntime;
464
465 if (cfs_rq->curr)
466 vruntime = cfs_rq->curr->vruntime;
467
468 if (cfs_rq->rb_leftmost) {
469 struct sched_entity *se = rb_entry(cfs_rq->rb_leftmost,
470 struct sched_entity,
471 run_node);
472
Peter Zijlstrae17036d2009-01-15 14:53:39 +0100473 if (!cfs_rq->curr)
Peter Zijlstra1af5f732008-10-24 11:06:13 +0200474 vruntime = se->vruntime;
475 else
476 vruntime = min_vruntime(vruntime, se->vruntime);
477 }
478
Andrei Epure1bf08232013-03-12 21:12:24 +0200479 /* ensure we never gain time by being placed backwards. */
Peter Zijlstra1af5f732008-10-24 11:06:13 +0200480 cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
Peter Zijlstra3fe16982011-04-05 17:23:48 +0200481#ifndef CONFIG_64BIT
482 smp_wmb();
483 cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
484#endif
Peter Zijlstra1af5f732008-10-24 11:06:13 +0200485}
486
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200487/*
488 * Enqueue an entity into the rb-tree:
489 */
Ingo Molnar0702e3e2007-10-15 17:00:14 +0200490static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200491{
492 struct rb_node **link = &cfs_rq->tasks_timeline.rb_node;
493 struct rb_node *parent = NULL;
494 struct sched_entity *entry;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200495 int leftmost = 1;
496
497 /*
498 * Find the right place in the rbtree:
499 */
500 while (*link) {
501 parent = *link;
502 entry = rb_entry(parent, struct sched_entity, run_node);
503 /*
504 * We dont care about collisions. Nodes with
505 * the same key stay together.
506 */
Stephan Baerwolf2bd2d6f2011-07-20 14:46:59 +0200507 if (entity_before(se, entry)) {
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200508 link = &parent->rb_left;
509 } else {
510 link = &parent->rb_right;
511 leftmost = 0;
512 }
513 }
514
515 /*
516 * Maintain a cache of leftmost tree entries (it is frequently
517 * used):
518 */
Peter Zijlstra1af5f732008-10-24 11:06:13 +0200519 if (leftmost)
Ingo Molnar57cb4992007-10-15 17:00:11 +0200520 cfs_rq->rb_leftmost = &se->run_node;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200521
522 rb_link_node(&se->run_node, parent, link);
523 rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200524}
525
Ingo Molnar0702e3e2007-10-15 17:00:14 +0200526static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200527{
Peter Zijlstra3fe69742008-03-14 20:55:51 +0100528 if (cfs_rq->rb_leftmost == &se->run_node) {
529 struct rb_node *next_node;
Peter Zijlstra3fe69742008-03-14 20:55:51 +0100530
531 next_node = rb_next(&se->run_node);
532 cfs_rq->rb_leftmost = next_node;
Peter Zijlstra3fe69742008-03-14 20:55:51 +0100533 }
Ingo Molnare9acbff2007-10-15 17:00:04 +0200534
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200535 rb_erase(&se->run_node, &cfs_rq->tasks_timeline);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200536}
537
Peter Zijlstra029632f2011-10-25 10:00:11 +0200538struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200539{
Peter Zijlstraf4b67552008-11-04 21:25:07 +0100540 struct rb_node *left = cfs_rq->rb_leftmost;
541
542 if (!left)
543 return NULL;
544
545 return rb_entry(left, struct sched_entity, run_node);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200546}
547
Rik van Rielac53db52011-02-01 09:51:03 -0500548static struct sched_entity *__pick_next_entity(struct sched_entity *se)
549{
550 struct rb_node *next = rb_next(&se->run_node);
551
552 if (!next)
553 return NULL;
554
555 return rb_entry(next, struct sched_entity, run_node);
556}
557
558#ifdef CONFIG_SCHED_DEBUG
Peter Zijlstra029632f2011-10-25 10:00:11 +0200559struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
Peter Zijlstraaeb73b02007-10-15 17:00:05 +0200560{
Ingo Molnar7eee3e62008-02-22 10:32:21 +0100561 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline);
Peter Zijlstraaeb73b02007-10-15 17:00:05 +0200562
Balbir Singh70eee742008-02-22 13:25:53 +0530563 if (!last)
564 return NULL;
Ingo Molnar7eee3e62008-02-22 10:32:21 +0100565
566 return rb_entry(last, struct sched_entity, run_node);
Peter Zijlstraaeb73b02007-10-15 17:00:05 +0200567}
568
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200569/**************************************************************
570 * Scheduling class statistics methods:
571 */
572
Christian Ehrhardtacb4a842009-11-30 12:16:48 +0100573int sched_proc_update_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -0700574 void __user *buffer, size_t *lenp,
Peter Zijlstrab2be5e92007-11-09 22:39:37 +0100575 loff_t *ppos)
576{
Alexey Dobriyan8d65af72009-09-23 15:57:19 -0700577 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
Christian Ehrhardtacb4a842009-11-30 12:16:48 +0100578 int factor = get_update_sysctl_factor();
Peter Zijlstrab2be5e92007-11-09 22:39:37 +0100579
580 if (ret || !write)
581 return ret;
582
583 sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
584 sysctl_sched_min_granularity);
585
Christian Ehrhardtacb4a842009-11-30 12:16:48 +0100586#define WRT_SYSCTL(name) \
587 (normalized_sysctl_##name = sysctl_##name / (factor))
588 WRT_SYSCTL(sched_min_granularity);
589 WRT_SYSCTL(sched_latency);
590 WRT_SYSCTL(sched_wakeup_granularity);
Christian Ehrhardtacb4a842009-11-30 12:16:48 +0100591#undef WRT_SYSCTL
592
Peter Zijlstrab2be5e92007-11-09 22:39:37 +0100593 return 0;
594}
595#endif
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200596
597/*
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200598 * delta /= w
Peter Zijlstraa7be37a2008-06-27 13:41:11 +0200599 */
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100600static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
Peter Zijlstraa7be37a2008-06-27 13:41:11 +0200601{
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200602 if (unlikely(se->load.weight != NICE_0_LOAD))
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100603 delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
Peter Zijlstraa7be37a2008-06-27 13:41:11 +0200604
605 return delta;
606}
607
608/*
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200609 * The idea is to set a period in which each task runs once.
610 *
Borislav Petkov532b1852012-08-08 16:16:04 +0200611 * When there are too many tasks (sched_nr_latency) we have to stretch
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200612 * this period because otherwise the slices get too small.
613 *
614 * p = (nr <= nl) ? l : l*nr/nl
615 */
Peter Zijlstra4d78e7b2007-10-15 17:00:04 +0200616static u64 __sched_period(unsigned long nr_running)
617{
618 u64 period = sysctl_sched_latency;
Peter Zijlstrab2be5e92007-11-09 22:39:37 +0100619 unsigned long nr_latency = sched_nr_latency;
Peter Zijlstra4d78e7b2007-10-15 17:00:04 +0200620
621 if (unlikely(nr_running > nr_latency)) {
Peter Zijlstra4bf0b772008-01-25 21:08:21 +0100622 period = sysctl_sched_min_granularity;
Peter Zijlstra4d78e7b2007-10-15 17:00:04 +0200623 period *= nr_running;
Peter Zijlstra4d78e7b2007-10-15 17:00:04 +0200624 }
625
626 return period;
627}
628
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200629/*
630 * We calculate the wall-time slice from the period by taking a part
631 * proportional to the weight.
632 *
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200633 * s = p*P[w/rw]
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200634 */
Peter Zijlstra6d0f0eb2007-10-15 17:00:05 +0200635static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
Peter Zijlstra21805082007-08-25 18:41:53 +0200636{
Mike Galbraith0a582442009-01-02 12:16:42 +0100637 u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq);
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200638
Mike Galbraith0a582442009-01-02 12:16:42 +0100639 for_each_sched_entity(se) {
Lin Ming6272d682009-01-15 17:17:15 +0100640 struct load_weight *load;
Christian Engelmayer3104bf02009-06-16 10:35:12 +0200641 struct load_weight lw;
Lin Ming6272d682009-01-15 17:17:15 +0100642
643 cfs_rq = cfs_rq_of(se);
644 load = &cfs_rq->load;
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200645
Mike Galbraith0a582442009-01-02 12:16:42 +0100646 if (unlikely(!se->on_rq)) {
Christian Engelmayer3104bf02009-06-16 10:35:12 +0200647 lw = cfs_rq->load;
Mike Galbraith0a582442009-01-02 12:16:42 +0100648
649 update_load_add(&lw, se->load.weight);
650 load = &lw;
651 }
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100652 slice = __calc_delta(slice, se->load.weight, load);
Mike Galbraith0a582442009-01-02 12:16:42 +0100653 }
654 return slice;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200655}
656
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200657/*
Andrei Epure660cc002013-03-11 12:03:20 +0200658 * We calculate the vruntime slice of a to-be-inserted task.
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200659 *
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200660 * vs = s/w
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200661 */
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200662static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnar647e7ca2007-10-15 17:00:13 +0200663{
Peter Zijlstraf9c0b092008-10-17 19:27:04 +0200664 return calc_delta_fair(sched_slice(cfs_rq, se), se);
Peter Zijlstraa7be37a2008-06-27 13:41:11 +0200665}
666
Alex Shia75cdaa2013-06-20 10:18:47 +0800667#ifdef CONFIG_SMP
Mel Gormanfb13c7e2013-10-07 11:29:17 +0100668static unsigned long task_h_load(struct task_struct *p);
669
Alex Shia75cdaa2013-06-20 10:18:47 +0800670static inline void __update_task_entity_contrib(struct sched_entity *se);
671
672/* Give new task start runnable values to heavy its load in infant time */
673void init_task_runnable_average(struct task_struct *p)
674{
675 u32 slice;
676
677 p->se.avg.decay_count = 0;
678 slice = sched_slice(task_cfs_rq(p), &p->se) >> 10;
679 p->se.avg.runnable_avg_sum = slice;
680 p->se.avg.runnable_avg_period = slice;
681 __update_task_entity_contrib(&p->se);
682}
683#else
684void init_task_runnable_average(struct task_struct *p)
685{
686}
687#endif
688
Peter Zijlstraa7be37a2008-06-27 13:41:11 +0200689/*
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100690 * Update the current task's runtime statistics.
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200691 */
Ingo Molnarb7cc0892007-08-09 11:16:47 +0200692static void update_curr(struct cfs_rq *cfs_rq)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200693{
Ingo Molnar429d43b2007-10-15 17:00:03 +0200694 struct sched_entity *curr = cfs_rq->curr;
Frederic Weisbecker78becc22013-04-12 01:51:02 +0200695 u64 now = rq_clock_task(rq_of(cfs_rq));
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100696 u64 delta_exec;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200697
698 if (unlikely(!curr))
699 return;
700
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100701 delta_exec = now - curr->exec_start;
702 if (unlikely((s64)delta_exec <= 0))
Peter Zijlstra34f28ec2008-12-16 08:45:31 +0100703 return;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200704
Ingo Molnar8ebc91d2007-10-15 17:00:03 +0200705 curr->exec_start = now;
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +0100706
Peter Zijlstra9dbdb152013-11-18 18:27:06 +0100707 schedstat_set(curr->statistics.exec_max,
708 max(delta_exec, curr->statistics.exec_max));
709
710 curr->sum_exec_runtime += delta_exec;
711 schedstat_add(cfs_rq, exec_clock, delta_exec);
712
713 curr->vruntime += calc_delta_fair(delta_exec, curr);
714 update_min_vruntime(cfs_rq);
715
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +0100716 if (entity_is_task(curr)) {
717 struct task_struct *curtask = task_of(curr);
718
Ingo Molnarf977bb42009-09-13 18:15:54 +0200719 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +0100720 cpuacct_charge(curtask, delta_exec);
Frank Mayharf06febc2008-09-12 09:54:39 -0700721 account_group_exec_runtime(curtask, delta_exec);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +0100722 }
Paul Turnerec12cb72011-07-21 09:43:30 -0700723
724 account_cfs_rq_runtime(cfs_rq, delta_exec);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200725}
726
727static inline void
Ingo Molnar5870db52007-08-09 11:16:47 +0200728update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200729{
Frederic Weisbecker78becc22013-04-12 01:51:02 +0200730 schedstat_set(se->statistics.wait_start, rq_clock(rq_of(cfs_rq)));
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200731}
732
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200733/*
734 * Task is being enqueued - update stats:
735 */
Ingo Molnard2417e52007-08-09 11:16:47 +0200736static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200737{
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200738 /*
739 * Are we enqueueing a waiting task? (for current tasks
740 * a dequeue/enqueue event is a NOP)
741 */
Ingo Molnar429d43b2007-10-15 17:00:03 +0200742 if (se != cfs_rq->curr)
Ingo Molnar5870db52007-08-09 11:16:47 +0200743 update_stats_wait_start(cfs_rq, se);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200744}
745
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200746static void
Ingo Molnar9ef0a962007-08-09 11:16:47 +0200747update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200748{
Lucas De Marchi41acab82010-03-10 23:37:45 -0300749 schedstat_set(se->statistics.wait_max, max(se->statistics.wait_max,
Frederic Weisbecker78becc22013-04-12 01:51:02 +0200750 rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start));
Lucas De Marchi41acab82010-03-10 23:37:45 -0300751 schedstat_set(se->statistics.wait_count, se->statistics.wait_count + 1);
752 schedstat_set(se->statistics.wait_sum, se->statistics.wait_sum +
Frederic Weisbecker78becc22013-04-12 01:51:02 +0200753 rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start);
Peter Zijlstra768d0c22009-07-23 20:13:26 +0200754#ifdef CONFIG_SCHEDSTATS
755 if (entity_is_task(se)) {
756 trace_sched_stat_wait(task_of(se),
Frederic Weisbecker78becc22013-04-12 01:51:02 +0200757 rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start);
Peter Zijlstra768d0c22009-07-23 20:13:26 +0200758 }
759#endif
Lucas De Marchi41acab82010-03-10 23:37:45 -0300760 schedstat_set(se->statistics.wait_start, 0);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200761}
762
763static inline void
Ingo Molnar19b6a2e2007-08-09 11:16:48 +0200764update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200765{
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200766 /*
767 * Mark the end of the wait period if dequeueing a
768 * waiting task:
769 */
Ingo Molnar429d43b2007-10-15 17:00:03 +0200770 if (se != cfs_rq->curr)
Ingo Molnar9ef0a962007-08-09 11:16:47 +0200771 update_stats_wait_end(cfs_rq, se);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200772}
773
774/*
775 * We are picking a new current task - update its stats:
776 */
777static inline void
Ingo Molnar79303e92007-08-09 11:16:47 +0200778update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200779{
780 /*
781 * We are starting a new run period:
782 */
Frederic Weisbecker78becc22013-04-12 01:51:02 +0200783 se->exec_start = rq_clock_task(rq_of(cfs_rq));
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200784}
785
Ingo Molnarbf0f6f22007-07-09 18:51:58 +0200786/**************************************************
787 * Scheduling class queueing methods:
788 */
789
Peter Zijlstracbee9f82012-10-25 14:16:43 +0200790#ifdef CONFIG_NUMA_BALANCING
791/*
Mel Gorman598f0ec2013-10-07 11:28:55 +0100792 * Approximate time to scan a full NUMA task in ms. The task scan period is
793 * calculated based on the tasks virtual memory size and
794 * numa_balancing_scan_size.
Peter Zijlstracbee9f82012-10-25 14:16:43 +0200795 */
Mel Gorman598f0ec2013-10-07 11:28:55 +0100796unsigned int sysctl_numa_balancing_scan_period_min = 1000;
797unsigned int sysctl_numa_balancing_scan_period_max = 60000;
Peter Zijlstra6e5fb222012-10-25 14:16:45 +0200798
799/* Portion of address space to scan in MB */
800unsigned int sysctl_numa_balancing_scan_size = 256;
Peter Zijlstracbee9f82012-10-25 14:16:43 +0200801
Peter Zijlstra4b96a292012-10-25 14:16:47 +0200802/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
803unsigned int sysctl_numa_balancing_scan_delay = 1000;
804
Mel Gorman598f0ec2013-10-07 11:28:55 +0100805static unsigned int task_nr_scan_windows(struct task_struct *p)
806{
807 unsigned long rss = 0;
808 unsigned long nr_scan_pages;
809
810 /*
811 * Calculations based on RSS as non-present and empty pages are skipped
812 * by the PTE scanner and NUMA hinting faults should be trapped based
813 * on resident pages
814 */
815 nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
816 rss = get_mm_rss(p->mm);
817 if (!rss)
818 rss = nr_scan_pages;
819
820 rss = round_up(rss, nr_scan_pages);
821 return rss / nr_scan_pages;
822}
823
824/* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
825#define MAX_SCAN_WINDOW 2560
826
827static unsigned int task_scan_min(struct task_struct *p)
828{
829 unsigned int scan, floor;
830 unsigned int windows = 1;
831
832 if (sysctl_numa_balancing_scan_size < MAX_SCAN_WINDOW)
833 windows = MAX_SCAN_WINDOW / sysctl_numa_balancing_scan_size;
834 floor = 1000 / windows;
835
836 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
837 return max_t(unsigned int, floor, scan);
838}
839
840static unsigned int task_scan_max(struct task_struct *p)
841{
842 unsigned int smin = task_scan_min(p);
843 unsigned int smax;
844
845 /* Watch for min being lower than max due to floor calculations */
846 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
847 return max(smin, smax);
848}
849
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +0100850static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
851{
852 rq->nr_numa_running += (p->numa_preferred_nid != -1);
853 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
854}
855
856static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
857{
858 rq->nr_numa_running -= (p->numa_preferred_nid != -1);
859 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
860}
861
Peter Zijlstra8c8a7432013-10-07 11:29:21 +0100862struct numa_group {
863 atomic_t refcount;
864
865 spinlock_t lock; /* nr_tasks, tasks */
866 int nr_tasks;
Mel Gormane29cf082013-10-07 11:29:22 +0100867 pid_t gid;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +0100868 struct list_head task_list;
869
870 struct rcu_head rcu;
Rik van Riel20e07de2014-01-27 17:03:43 -0500871 nodemask_t active_nodes;
Mel Gorman989348b2013-10-07 11:29:40 +0100872 unsigned long total_faults;
Rik van Riel7e2703e2014-01-27 17:03:45 -0500873 /*
874 * Faults_cpu is used to decide whether memory should move
875 * towards the CPU. As a consequence, these stats are weighted
876 * more by CPU use than by memory faults.
877 */
Rik van Riel50ec8a42014-01-27 17:03:42 -0500878 unsigned long *faults_cpu;
Mel Gorman989348b2013-10-07 11:29:40 +0100879 unsigned long faults[0];
Peter Zijlstra8c8a7432013-10-07 11:29:21 +0100880};
881
Rik van Rielbe1e4e72014-01-27 17:03:48 -0500882/* Shared or private faults. */
883#define NR_NUMA_HINT_FAULT_TYPES 2
884
885/* Memory and CPU locality */
886#define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
887
888/* Averaged statistics, and temporary buffers. */
889#define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
890
Mel Gormane29cf082013-10-07 11:29:22 +0100891pid_t task_numa_group_id(struct task_struct *p)
892{
893 return p->numa_group ? p->numa_group->gid : 0;
894}
895
Mel Gormanac8e8952013-10-07 11:29:03 +0100896static inline int task_faults_idx(int nid, int priv)
897{
Rik van Rielbe1e4e72014-01-27 17:03:48 -0500898 return NR_NUMA_HINT_FAULT_TYPES * nid + priv;
Mel Gormanac8e8952013-10-07 11:29:03 +0100899}
900
901static inline unsigned long task_faults(struct task_struct *p, int nid)
902{
Rik van Rielff1df892014-01-27 17:03:41 -0500903 if (!p->numa_faults_memory)
Mel Gormanac8e8952013-10-07 11:29:03 +0100904 return 0;
905
Rik van Rielff1df892014-01-27 17:03:41 -0500906 return p->numa_faults_memory[task_faults_idx(nid, 0)] +
907 p->numa_faults_memory[task_faults_idx(nid, 1)];
Mel Gormanac8e8952013-10-07 11:29:03 +0100908}
909
Mel Gorman83e1d2c2013-10-07 11:29:27 +0100910static inline unsigned long group_faults(struct task_struct *p, int nid)
911{
912 if (!p->numa_group)
913 return 0;
914
Wanpeng Li82897b42013-12-12 15:23:25 +0800915 return p->numa_group->faults[task_faults_idx(nid, 0)] +
916 p->numa_group->faults[task_faults_idx(nid, 1)];
Mel Gorman83e1d2c2013-10-07 11:29:27 +0100917}
918
Rik van Riel20e07de2014-01-27 17:03:43 -0500919static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
920{
921 return group->faults_cpu[task_faults_idx(nid, 0)] +
922 group->faults_cpu[task_faults_idx(nid, 1)];
923}
924
Mel Gorman83e1d2c2013-10-07 11:29:27 +0100925/*
926 * These return the fraction of accesses done by a particular task, or
927 * task group, on a particular numa node. The group weight is given a
928 * larger multiplier, in order to group tasks together that are almost
929 * evenly spread out between numa nodes.
930 */
931static inline unsigned long task_weight(struct task_struct *p, int nid)
932{
933 unsigned long total_faults;
934
Rik van Rielff1df892014-01-27 17:03:41 -0500935 if (!p->numa_faults_memory)
Mel Gorman83e1d2c2013-10-07 11:29:27 +0100936 return 0;
937
938 total_faults = p->total_numa_faults;
939
940 if (!total_faults)
941 return 0;
942
943 return 1000 * task_faults(p, nid) / total_faults;
944}
945
946static inline unsigned long group_weight(struct task_struct *p, int nid)
947{
Mel Gorman989348b2013-10-07 11:29:40 +0100948 if (!p->numa_group || !p->numa_group->total_faults)
Mel Gorman83e1d2c2013-10-07 11:29:27 +0100949 return 0;
950
Mel Gorman989348b2013-10-07 11:29:40 +0100951 return 1000 * group_faults(p, nid) / p->numa_group->total_faults;
Mel Gorman83e1d2c2013-10-07 11:29:27 +0100952}
953
Rik van Riel10f39042014-01-27 17:03:44 -0500954bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
955 int src_nid, int dst_cpu)
956{
957 struct numa_group *ng = p->numa_group;
958 int dst_nid = cpu_to_node(dst_cpu);
959 int last_cpupid, this_cpupid;
960
961 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
962
963 /*
964 * Multi-stage node selection is used in conjunction with a periodic
965 * migration fault to build a temporal task<->page relation. By using
966 * a two-stage filter we remove short/unlikely relations.
967 *
968 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
969 * a task's usage of a particular page (n_p) per total usage of this
970 * page (n_t) (in a given time-span) to a probability.
971 *
972 * Our periodic faults will sample this probability and getting the
973 * same result twice in a row, given these samples are fully
974 * independent, is then given by P(n)^2, provided our sample period
975 * is sufficiently short compared to the usage pattern.
976 *
977 * This quadric squishes small probabilities, making it less likely we
978 * act on an unlikely task<->page relation.
979 */
980 last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
981 if (!cpupid_pid_unset(last_cpupid) &&
982 cpupid_to_nid(last_cpupid) != dst_nid)
983 return false;
984
985 /* Always allow migrate on private faults */
986 if (cpupid_match_pid(p, last_cpupid))
987 return true;
988
989 /* A shared fault, but p->numa_group has not been set up yet. */
990 if (!ng)
991 return true;
992
993 /*
994 * Do not migrate if the destination is not a node that
995 * is actively used by this numa group.
996 */
997 if (!node_isset(dst_nid, ng->active_nodes))
998 return false;
999
1000 /*
1001 * Source is a node that is not actively used by this
1002 * numa group, while the destination is. Migrate.
1003 */
1004 if (!node_isset(src_nid, ng->active_nodes))
1005 return true;
1006
1007 /*
1008 * Both source and destination are nodes in active
1009 * use by this numa group. Maximize memory bandwidth
1010 * by migrating from more heavily used groups, to less
1011 * heavily used ones, spreading the load around.
1012 * Use a 1/4 hysteresis to avoid spurious page movement.
1013 */
1014 return group_faults(p, dst_nid) < (group_faults(p, src_nid) * 3 / 4);
1015}
1016
Mel Gormane6628d52013-10-07 11:29:02 +01001017static unsigned long weighted_cpuload(const int cpu);
Mel Gorman58d081b2013-10-07 11:29:10 +01001018static unsigned long source_load(int cpu, int type);
1019static unsigned long target_load(int cpu, int type);
1020static unsigned long power_of(int cpu);
1021static long effective_load(struct task_group *tg, int cpu, long wl, long wg);
Mel Gormane6628d52013-10-07 11:29:02 +01001022
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001023/* Cached statistics for all CPUs within a node */
Mel Gorman58d081b2013-10-07 11:29:10 +01001024struct numa_stats {
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001025 unsigned long nr_running;
Mel Gorman58d081b2013-10-07 11:29:10 +01001026 unsigned long load;
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001027
1028 /* Total compute capacity of CPUs on a node */
1029 unsigned long power;
1030
1031 /* Approximate capacity in terms of runnable tasks on a node */
1032 unsigned long capacity;
1033 int has_capacity;
Mel Gorman58d081b2013-10-07 11:29:10 +01001034};
Mel Gormane6628d52013-10-07 11:29:02 +01001035
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001036/*
1037 * XXX borrowed from update_sg_lb_stats
1038 */
1039static void update_numa_stats(struct numa_stats *ns, int nid)
1040{
Peter Zijlstra5eca82a2013-11-06 18:47:57 +01001041 int cpu, cpus = 0;
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001042
1043 memset(ns, 0, sizeof(*ns));
1044 for_each_cpu(cpu, cpumask_of_node(nid)) {
1045 struct rq *rq = cpu_rq(cpu);
1046
1047 ns->nr_running += rq->nr_running;
1048 ns->load += weighted_cpuload(cpu);
1049 ns->power += power_of(cpu);
Peter Zijlstra5eca82a2013-11-06 18:47:57 +01001050
1051 cpus++;
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001052 }
1053
Peter Zijlstra5eca82a2013-11-06 18:47:57 +01001054 /*
1055 * If we raced with hotplug and there are no CPUs left in our mask
1056 * the @ns structure is NULL'ed and task_numa_compare() will
1057 * not find this node attractive.
1058 *
1059 * We'll either bail at !has_capacity, or we'll detect a huge imbalance
1060 * and bail there.
1061 */
1062 if (!cpus)
1063 return;
1064
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001065 ns->load = (ns->load * SCHED_POWER_SCALE) / ns->power;
1066 ns->capacity = DIV_ROUND_CLOSEST(ns->power, SCHED_POWER_SCALE);
1067 ns->has_capacity = (ns->nr_running < ns->capacity);
1068}
1069
Mel Gorman58d081b2013-10-07 11:29:10 +01001070struct task_numa_env {
1071 struct task_struct *p;
1072
1073 int src_cpu, src_nid;
1074 int dst_cpu, dst_nid;
1075
1076 struct numa_stats src_stats, dst_stats;
1077
Wanpeng Li40ea2b42013-12-05 19:10:17 +08001078 int imbalance_pct;
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001079
1080 struct task_struct *best_task;
1081 long best_imp;
Mel Gorman58d081b2013-10-07 11:29:10 +01001082 int best_cpu;
1083};
1084
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001085static void task_numa_assign(struct task_numa_env *env,
1086 struct task_struct *p, long imp)
1087{
1088 if (env->best_task)
1089 put_task_struct(env->best_task);
1090 if (p)
1091 get_task_struct(p);
1092
1093 env->best_task = p;
1094 env->best_imp = imp;
1095 env->best_cpu = env->dst_cpu;
1096}
1097
1098/*
1099 * This checks if the overall compute and NUMA accesses of the system would
1100 * be improved if the source tasks was migrated to the target dst_cpu taking
1101 * into account that it might be best if task running on the dst_cpu should
1102 * be exchanged with the source task
1103 */
Rik van Riel887c2902013-10-07 11:29:31 +01001104static void task_numa_compare(struct task_numa_env *env,
1105 long taskimp, long groupimp)
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001106{
1107 struct rq *src_rq = cpu_rq(env->src_cpu);
1108 struct rq *dst_rq = cpu_rq(env->dst_cpu);
1109 struct task_struct *cur;
1110 long dst_load, src_load;
1111 long load;
Rik van Riel887c2902013-10-07 11:29:31 +01001112 long imp = (groupimp > 0) ? groupimp : taskimp;
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001113
1114 rcu_read_lock();
1115 cur = ACCESS_ONCE(dst_rq->curr);
1116 if (cur->pid == 0) /* idle */
1117 cur = NULL;
1118
1119 /*
1120 * "imp" is the fault differential for the source task between the
1121 * source and destination node. Calculate the total differential for
1122 * the source task and potential destination task. The more negative
1123 * the value is, the more rmeote accesses that would be expected to
1124 * be incurred if the tasks were swapped.
1125 */
1126 if (cur) {
1127 /* Skip this swap candidate if cannot move to the source cpu */
1128 if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur)))
1129 goto unlock;
1130
Rik van Riel887c2902013-10-07 11:29:31 +01001131 /*
1132 * If dst and source tasks are in the same NUMA group, or not
Rik van Rielca28aa532013-10-07 11:29:32 +01001133 * in any group then look only at task weights.
Rik van Riel887c2902013-10-07 11:29:31 +01001134 */
Rik van Rielca28aa532013-10-07 11:29:32 +01001135 if (cur->numa_group == env->p->numa_group) {
Rik van Riel887c2902013-10-07 11:29:31 +01001136 imp = taskimp + task_weight(cur, env->src_nid) -
1137 task_weight(cur, env->dst_nid);
Rik van Rielca28aa532013-10-07 11:29:32 +01001138 /*
1139 * Add some hysteresis to prevent swapping the
1140 * tasks within a group over tiny differences.
1141 */
1142 if (cur->numa_group)
1143 imp -= imp/16;
Rik van Riel887c2902013-10-07 11:29:31 +01001144 } else {
Rik van Rielca28aa532013-10-07 11:29:32 +01001145 /*
1146 * Compare the group weights. If a task is all by
1147 * itself (not part of a group), use the task weight
1148 * instead.
1149 */
1150 if (env->p->numa_group)
1151 imp = groupimp;
1152 else
1153 imp = taskimp;
1154
1155 if (cur->numa_group)
1156 imp += group_weight(cur, env->src_nid) -
1157 group_weight(cur, env->dst_nid);
1158 else
1159 imp += task_weight(cur, env->src_nid) -
1160 task_weight(cur, env->dst_nid);
Rik van Riel887c2902013-10-07 11:29:31 +01001161 }
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001162 }
1163
1164 if (imp < env->best_imp)
1165 goto unlock;
1166
1167 if (!cur) {
1168 /* Is there capacity at our destination? */
1169 if (env->src_stats.has_capacity &&
1170 !env->dst_stats.has_capacity)
1171 goto unlock;
1172
1173 goto balance;
1174 }
1175
1176 /* Balance doesn't matter much if we're running a task per cpu */
1177 if (src_rq->nr_running == 1 && dst_rq->nr_running == 1)
1178 goto assign;
1179
1180 /*
1181 * In the overloaded case, try and keep the load balanced.
1182 */
1183balance:
1184 dst_load = env->dst_stats.load;
1185 src_load = env->src_stats.load;
1186
1187 /* XXX missing power terms */
1188 load = task_h_load(env->p);
1189 dst_load += load;
1190 src_load -= load;
1191
1192 if (cur) {
1193 load = task_h_load(cur);
1194 dst_load -= load;
1195 src_load += load;
1196 }
1197
1198 /* make src_load the smaller */
1199 if (dst_load < src_load)
1200 swap(dst_load, src_load);
1201
1202 if (src_load * env->imbalance_pct < dst_load * 100)
1203 goto unlock;
1204
1205assign:
1206 task_numa_assign(env, cur, imp);
1207unlock:
1208 rcu_read_unlock();
1209}
1210
Rik van Riel887c2902013-10-07 11:29:31 +01001211static void task_numa_find_cpu(struct task_numa_env *env,
1212 long taskimp, long groupimp)
Mel Gorman2c8a50a2013-10-07 11:29:18 +01001213{
1214 int cpu;
1215
1216 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
1217 /* Skip this CPU if the source task cannot migrate */
1218 if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p)))
1219 continue;
1220
1221 env->dst_cpu = cpu;
Rik van Riel887c2902013-10-07 11:29:31 +01001222 task_numa_compare(env, taskimp, groupimp);
Mel Gorman2c8a50a2013-10-07 11:29:18 +01001223 }
1224}
1225
Mel Gorman58d081b2013-10-07 11:29:10 +01001226static int task_numa_migrate(struct task_struct *p)
Mel Gormane6628d52013-10-07 11:29:02 +01001227{
Mel Gorman58d081b2013-10-07 11:29:10 +01001228 struct task_numa_env env = {
1229 .p = p,
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001230
Mel Gorman58d081b2013-10-07 11:29:10 +01001231 .src_cpu = task_cpu(p),
Ingo Molnarb32e86b2013-10-07 11:29:30 +01001232 .src_nid = task_node(p),
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001233
1234 .imbalance_pct = 112,
1235
1236 .best_task = NULL,
1237 .best_imp = 0,
1238 .best_cpu = -1
Mel Gorman58d081b2013-10-07 11:29:10 +01001239 };
1240 struct sched_domain *sd;
Rik van Riel887c2902013-10-07 11:29:31 +01001241 unsigned long taskweight, groupweight;
Mel Gorman2c8a50a2013-10-07 11:29:18 +01001242 int nid, ret;
Rik van Riel887c2902013-10-07 11:29:31 +01001243 long taskimp, groupimp;
Mel Gormane6628d52013-10-07 11:29:02 +01001244
Mel Gorman58d081b2013-10-07 11:29:10 +01001245 /*
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001246 * Pick the lowest SD_NUMA domain, as that would have the smallest
1247 * imbalance and would be the first to start moving tasks about.
1248 *
1249 * And we want to avoid any moving of tasks about, as that would create
1250 * random movement of tasks -- counter the numa conditions we're trying
1251 * to satisfy here.
Mel Gorman58d081b2013-10-07 11:29:10 +01001252 */
Mel Gormane6628d52013-10-07 11:29:02 +01001253 rcu_read_lock();
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001254 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
Rik van Riel46a73e82013-11-11 19:29:25 -05001255 if (sd)
1256 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
Mel Gormane6628d52013-10-07 11:29:02 +01001257 rcu_read_unlock();
1258
Rik van Riel46a73e82013-11-11 19:29:25 -05001259 /*
1260 * Cpusets can break the scheduler domain tree into smaller
1261 * balance domains, some of which do not cross NUMA boundaries.
1262 * Tasks that are "trapped" in such domains cannot be migrated
1263 * elsewhere, so there is no point in (re)trying.
1264 */
1265 if (unlikely(!sd)) {
Wanpeng Lide1b3012013-12-12 15:23:24 +08001266 p->numa_preferred_nid = task_node(p);
Rik van Riel46a73e82013-11-11 19:29:25 -05001267 return -EINVAL;
1268 }
1269
Rik van Riel887c2902013-10-07 11:29:31 +01001270 taskweight = task_weight(p, env.src_nid);
1271 groupweight = group_weight(p, env.src_nid);
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001272 update_numa_stats(&env.src_stats, env.src_nid);
Mel Gorman2c8a50a2013-10-07 11:29:18 +01001273 env.dst_nid = p->numa_preferred_nid;
Rik van Riel887c2902013-10-07 11:29:31 +01001274 taskimp = task_weight(p, env.dst_nid) - taskweight;
1275 groupimp = group_weight(p, env.dst_nid) - groupweight;
Mel Gorman2c8a50a2013-10-07 11:29:18 +01001276 update_numa_stats(&env.dst_stats, env.dst_nid);
Mel Gorman58d081b2013-10-07 11:29:10 +01001277
Rik van Riele1dda8a2013-10-07 11:29:19 +01001278 /* If the preferred nid has capacity, try to use it. */
1279 if (env.dst_stats.has_capacity)
Rik van Riel887c2902013-10-07 11:29:31 +01001280 task_numa_find_cpu(&env, taskimp, groupimp);
Rik van Riele1dda8a2013-10-07 11:29:19 +01001281
1282 /* No space available on the preferred nid. Look elsewhere. */
1283 if (env.best_cpu == -1) {
Mel Gorman2c8a50a2013-10-07 11:29:18 +01001284 for_each_online_node(nid) {
1285 if (nid == env.src_nid || nid == p->numa_preferred_nid)
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001286 continue;
1287
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001288 /* Only consider nodes where both task and groups benefit */
Rik van Riel887c2902013-10-07 11:29:31 +01001289 taskimp = task_weight(p, nid) - taskweight;
1290 groupimp = group_weight(p, nid) - groupweight;
1291 if (taskimp < 0 && groupimp < 0)
Mel Gorman2c8a50a2013-10-07 11:29:18 +01001292 continue;
1293
1294 env.dst_nid = nid;
1295 update_numa_stats(&env.dst_stats, env.dst_nid);
Rik van Riel887c2902013-10-07 11:29:31 +01001296 task_numa_find_cpu(&env, taskimp, groupimp);
Mel Gorman58d081b2013-10-07 11:29:10 +01001297 }
1298 }
1299
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001300 /* No better CPU than the current one was found. */
1301 if (env.best_cpu == -1)
1302 return -EAGAIN;
1303
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01001304 sched_setnuma(p, env.dst_nid);
1305
Rik van Riel04bb2f92013-10-07 11:29:36 +01001306 /*
1307 * Reset the scan period if the task is being rescheduled on an
1308 * alternative node to recheck if the tasks is now properly placed.
1309 */
1310 p->numa_scan_period = task_scan_min(p);
1311
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001312 if (env.best_task == NULL) {
Mel Gorman286549d2014-01-21 15:51:03 -08001313 ret = migrate_task_to(p, env.best_cpu);
1314 if (ret != 0)
1315 trace_sched_stick_numa(p, env.src_cpu, env.best_cpu);
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001316 return ret;
1317 }
1318
1319 ret = migrate_swap(p, env.best_task);
Mel Gorman286549d2014-01-21 15:51:03 -08001320 if (ret != 0)
1321 trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task));
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001322 put_task_struct(env.best_task);
1323 return ret;
Mel Gormane6628d52013-10-07 11:29:02 +01001324}
1325
Mel Gorman6b9a7462013-10-07 11:29:11 +01001326/* Attempt to migrate a task to a CPU on the preferred node. */
1327static void numa_migrate_preferred(struct task_struct *p)
1328{
Rik van Riel5085e2a2014-04-11 13:00:28 -04001329 unsigned long interval = HZ;
1330
Rik van Riel2739d3e2013-10-07 11:29:41 +01001331 /* This task has no NUMA fault statistics yet */
Rik van Rielff1df892014-01-27 17:03:41 -05001332 if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults_memory))
Rik van Riel2739d3e2013-10-07 11:29:41 +01001333 return;
1334
1335 /* Periodically retry migrating the task to the preferred node */
Rik van Riel5085e2a2014-04-11 13:00:28 -04001336 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
1337 p->numa_migrate_retry = jiffies + interval;
Rik van Riel2739d3e2013-10-07 11:29:41 +01001338
Mel Gorman6b9a7462013-10-07 11:29:11 +01001339 /* Success if task is already running on preferred CPU */
Wanpeng Lide1b3012013-12-12 15:23:24 +08001340 if (task_node(p) == p->numa_preferred_nid)
Mel Gorman6b9a7462013-10-07 11:29:11 +01001341 return;
1342
Mel Gorman6b9a7462013-10-07 11:29:11 +01001343 /* Otherwise, try migrate to a CPU on the preferred node */
Rik van Riel2739d3e2013-10-07 11:29:41 +01001344 task_numa_migrate(p);
Mel Gorman6b9a7462013-10-07 11:29:11 +01001345}
1346
Rik van Riel04bb2f92013-10-07 11:29:36 +01001347/*
Rik van Riel20e07de2014-01-27 17:03:43 -05001348 * Find the nodes on which the workload is actively running. We do this by
1349 * tracking the nodes from which NUMA hinting faults are triggered. This can
1350 * be different from the set of nodes where the workload's memory is currently
1351 * located.
1352 *
1353 * The bitmask is used to make smarter decisions on when to do NUMA page
1354 * migrations, To prevent flip-flopping, and excessive page migrations, nodes
1355 * are added when they cause over 6/16 of the maximum number of faults, but
1356 * only removed when they drop below 3/16.
1357 */
1358static void update_numa_active_node_mask(struct numa_group *numa_group)
1359{
1360 unsigned long faults, max_faults = 0;
1361 int nid;
1362
1363 for_each_online_node(nid) {
1364 faults = group_faults_cpu(numa_group, nid);
1365 if (faults > max_faults)
1366 max_faults = faults;
1367 }
1368
1369 for_each_online_node(nid) {
1370 faults = group_faults_cpu(numa_group, nid);
1371 if (!node_isset(nid, numa_group->active_nodes)) {
1372 if (faults > max_faults * 6 / 16)
1373 node_set(nid, numa_group->active_nodes);
1374 } else if (faults < max_faults * 3 / 16)
1375 node_clear(nid, numa_group->active_nodes);
1376 }
1377}
1378
1379/*
Rik van Riel04bb2f92013-10-07 11:29:36 +01001380 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
1381 * increments. The more local the fault statistics are, the higher the scan
1382 * period will be for the next scan window. If local/remote ratio is below
1383 * NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) the
1384 * scan period will decrease
1385 */
1386#define NUMA_PERIOD_SLOTS 10
1387#define NUMA_PERIOD_THRESHOLD 3
1388
1389/*
1390 * Increase the scan period (slow down scanning) if the majority of
1391 * our memory is already on our local node, or if the majority of
1392 * the page accesses are shared with other processes.
1393 * Otherwise, decrease the scan period.
1394 */
1395static void update_task_scan_period(struct task_struct *p,
1396 unsigned long shared, unsigned long private)
1397{
1398 unsigned int period_slot;
1399 int ratio;
1400 int diff;
1401
1402 unsigned long remote = p->numa_faults_locality[0];
1403 unsigned long local = p->numa_faults_locality[1];
1404
1405 /*
1406 * If there were no record hinting faults then either the task is
1407 * completely idle or all activity is areas that are not of interest
1408 * to automatic numa balancing. Scan slower
1409 */
1410 if (local + shared == 0) {
1411 p->numa_scan_period = min(p->numa_scan_period_max,
1412 p->numa_scan_period << 1);
1413
1414 p->mm->numa_next_scan = jiffies +
1415 msecs_to_jiffies(p->numa_scan_period);
1416
1417 return;
1418 }
1419
1420 /*
1421 * Prepare to scale scan period relative to the current period.
1422 * == NUMA_PERIOD_THRESHOLD scan period stays the same
1423 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
1424 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
1425 */
1426 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
1427 ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
1428 if (ratio >= NUMA_PERIOD_THRESHOLD) {
1429 int slot = ratio - NUMA_PERIOD_THRESHOLD;
1430 if (!slot)
1431 slot = 1;
1432 diff = slot * period_slot;
1433 } else {
1434 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
1435
1436 /*
1437 * Scale scan rate increases based on sharing. There is an
1438 * inverse relationship between the degree of sharing and
1439 * the adjustment made to the scanning period. Broadly
1440 * speaking the intent is that there is little point
1441 * scanning faster if shared accesses dominate as it may
1442 * simply bounce migrations uselessly
1443 */
Rik van Riel04bb2f92013-10-07 11:29:36 +01001444 ratio = DIV_ROUND_UP(private * NUMA_PERIOD_SLOTS, (private + shared));
1445 diff = (diff * ratio) / NUMA_PERIOD_SLOTS;
1446 }
1447
1448 p->numa_scan_period = clamp(p->numa_scan_period + diff,
1449 task_scan_min(p), task_scan_max(p));
1450 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
1451}
1452
Rik van Riel7e2703e2014-01-27 17:03:45 -05001453/*
1454 * Get the fraction of time the task has been running since the last
1455 * NUMA placement cycle. The scheduler keeps similar statistics, but
1456 * decays those on a 32ms period, which is orders of magnitude off
1457 * from the dozens-of-seconds NUMA balancing period. Use the scheduler
1458 * stats only if the task is so new there are no NUMA statistics yet.
1459 */
1460static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
1461{
1462 u64 runtime, delta, now;
1463 /* Use the start of this time slice to avoid calculations. */
1464 now = p->se.exec_start;
1465 runtime = p->se.sum_exec_runtime;
1466
1467 if (p->last_task_numa_placement) {
1468 delta = runtime - p->last_sum_exec_runtime;
1469 *period = now - p->last_task_numa_placement;
1470 } else {
1471 delta = p->se.avg.runnable_avg_sum;
1472 *period = p->se.avg.runnable_avg_period;
1473 }
1474
1475 p->last_sum_exec_runtime = runtime;
1476 p->last_task_numa_placement = now;
1477
1478 return delta;
1479}
1480
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001481static void task_numa_placement(struct task_struct *p)
1482{
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001483 int seq, nid, max_nid = -1, max_group_nid = -1;
1484 unsigned long max_faults = 0, max_group_faults = 0;
Rik van Riel04bb2f92013-10-07 11:29:36 +01001485 unsigned long fault_types[2] = { 0, 0 };
Rik van Riel7e2703e2014-01-27 17:03:45 -05001486 unsigned long total_faults;
1487 u64 runtime, period;
Mel Gorman7dbd13e2013-10-07 11:29:29 +01001488 spinlock_t *group_lock = NULL;
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001489
Hugh Dickins2832bc12012-12-19 17:42:16 -08001490 seq = ACCESS_ONCE(p->mm->numa_scan_seq);
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001491 if (p->numa_scan_seq == seq)
1492 return;
1493 p->numa_scan_seq = seq;
Mel Gorman598f0ec2013-10-07 11:28:55 +01001494 p->numa_scan_period_max = task_scan_max(p);
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001495
Rik van Riel7e2703e2014-01-27 17:03:45 -05001496 total_faults = p->numa_faults_locality[0] +
1497 p->numa_faults_locality[1];
1498 runtime = numa_get_avg_runtime(p, &period);
1499
Mel Gorman7dbd13e2013-10-07 11:29:29 +01001500 /* If the task is part of a group prevent parallel updates to group stats */
1501 if (p->numa_group) {
1502 group_lock = &p->numa_group->lock;
Mike Galbraith60e69ee2014-04-07 10:55:15 +02001503 spin_lock_irq(group_lock);
Mel Gorman7dbd13e2013-10-07 11:29:29 +01001504 }
1505
Mel Gorman688b7582013-10-07 11:28:58 +01001506 /* Find the node with the highest number of faults */
1507 for_each_online_node(nid) {
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001508 unsigned long faults = 0, group_faults = 0;
Mel Gormanac8e8952013-10-07 11:29:03 +01001509 int priv, i;
Mel Gorman745d6142013-10-07 11:28:59 +01001510
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001511 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
Rik van Riel7e2703e2014-01-27 17:03:45 -05001512 long diff, f_diff, f_weight;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001513
Mel Gormanac8e8952013-10-07 11:29:03 +01001514 i = task_faults_idx(nid, priv);
Mel Gorman745d6142013-10-07 11:28:59 +01001515
Mel Gormanac8e8952013-10-07 11:29:03 +01001516 /* Decay existing window, copy faults since last scan */
Rik van Riel35664fd2014-01-27 17:03:46 -05001517 diff = p->numa_faults_buffer_memory[i] - p->numa_faults_memory[i] / 2;
Rik van Rielff1df892014-01-27 17:03:41 -05001518 fault_types[priv] += p->numa_faults_buffer_memory[i];
1519 p->numa_faults_buffer_memory[i] = 0;
Mel Gormanfb13c7e2013-10-07 11:29:17 +01001520
Rik van Riel7e2703e2014-01-27 17:03:45 -05001521 /*
1522 * Normalize the faults_from, so all tasks in a group
1523 * count according to CPU use, instead of by the raw
1524 * number of faults. Tasks with little runtime have
1525 * little over-all impact on throughput, and thus their
1526 * faults are less important.
1527 */
1528 f_weight = div64_u64(runtime << 16, period + 1);
1529 f_weight = (f_weight * p->numa_faults_buffer_cpu[i]) /
1530 (total_faults + 1);
Rik van Riel35664fd2014-01-27 17:03:46 -05001531 f_diff = f_weight - p->numa_faults_cpu[i] / 2;
Rik van Riel50ec8a42014-01-27 17:03:42 -05001532 p->numa_faults_buffer_cpu[i] = 0;
1533
Rik van Riel35664fd2014-01-27 17:03:46 -05001534 p->numa_faults_memory[i] += diff;
1535 p->numa_faults_cpu[i] += f_diff;
Rik van Rielff1df892014-01-27 17:03:41 -05001536 faults += p->numa_faults_memory[i];
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001537 p->total_numa_faults += diff;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001538 if (p->numa_group) {
1539 /* safe because we can only change our own group */
Mel Gorman989348b2013-10-07 11:29:40 +01001540 p->numa_group->faults[i] += diff;
Rik van Riel50ec8a42014-01-27 17:03:42 -05001541 p->numa_group->faults_cpu[i] += f_diff;
Mel Gorman989348b2013-10-07 11:29:40 +01001542 p->numa_group->total_faults += diff;
1543 group_faults += p->numa_group->faults[i];
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001544 }
Mel Gormanac8e8952013-10-07 11:29:03 +01001545 }
1546
Mel Gorman688b7582013-10-07 11:28:58 +01001547 if (faults > max_faults) {
1548 max_faults = faults;
1549 max_nid = nid;
1550 }
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001551
1552 if (group_faults > max_group_faults) {
1553 max_group_faults = group_faults;
1554 max_group_nid = nid;
1555 }
1556 }
1557
Rik van Riel04bb2f92013-10-07 11:29:36 +01001558 update_task_scan_period(p, fault_types[0], fault_types[1]);
1559
Mel Gorman7dbd13e2013-10-07 11:29:29 +01001560 if (p->numa_group) {
Rik van Riel20e07de2014-01-27 17:03:43 -05001561 update_numa_active_node_mask(p->numa_group);
Mel Gorman7dbd13e2013-10-07 11:29:29 +01001562 /*
1563 * If the preferred task and group nids are different,
1564 * iterate over the nodes again to find the best place.
1565 */
1566 if (max_nid != max_group_nid) {
1567 unsigned long weight, max_weight = 0;
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001568
Mel Gorman7dbd13e2013-10-07 11:29:29 +01001569 for_each_online_node(nid) {
1570 weight = task_weight(p, nid) + group_weight(p, nid);
1571 if (weight > max_weight) {
1572 max_weight = weight;
1573 max_nid = nid;
1574 }
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001575 }
1576 }
Mel Gorman7dbd13e2013-10-07 11:29:29 +01001577
Mike Galbraith60e69ee2014-04-07 10:55:15 +02001578 spin_unlock_irq(group_lock);
Mel Gorman688b7582013-10-07 11:28:58 +01001579 }
1580
Mel Gorman6b9a7462013-10-07 11:29:11 +01001581 /* Preferred node as the node with the most faults */
Mel Gorman3a7053b2013-10-07 11:29:00 +01001582 if (max_faults && max_nid != p->numa_preferred_nid) {
Mel Gormane6628d52013-10-07 11:29:02 +01001583 /* Update the preferred nid and migrate task if possible */
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01001584 sched_setnuma(p, max_nid);
Mel Gorman6b9a7462013-10-07 11:29:11 +01001585 numa_migrate_preferred(p);
Mel Gorman3a7053b2013-10-07 11:29:00 +01001586 }
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001587}
1588
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001589static inline int get_numa_group(struct numa_group *grp)
1590{
1591 return atomic_inc_not_zero(&grp->refcount);
1592}
1593
1594static inline void put_numa_group(struct numa_group *grp)
1595{
1596 if (atomic_dec_and_test(&grp->refcount))
1597 kfree_rcu(grp, rcu);
1598}
1599
Mel Gorman3e6a9412013-10-07 11:29:35 +01001600static void task_numa_group(struct task_struct *p, int cpupid, int flags,
1601 int *priv)
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001602{
1603 struct numa_group *grp, *my_grp;
1604 struct task_struct *tsk;
1605 bool join = false;
1606 int cpu = cpupid_to_cpu(cpupid);
1607 int i;
1608
1609 if (unlikely(!p->numa_group)) {
1610 unsigned int size = sizeof(struct numa_group) +
Rik van Riel50ec8a42014-01-27 17:03:42 -05001611 4*nr_node_ids*sizeof(unsigned long);
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001612
1613 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1614 if (!grp)
1615 return;
1616
1617 atomic_set(&grp->refcount, 1);
1618 spin_lock_init(&grp->lock);
1619 INIT_LIST_HEAD(&grp->task_list);
Mel Gormane29cf082013-10-07 11:29:22 +01001620 grp->gid = p->pid;
Rik van Riel50ec8a42014-01-27 17:03:42 -05001621 /* Second half of the array tracks nids where faults happen */
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001622 grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES *
1623 nr_node_ids;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001624
Rik van Riel20e07de2014-01-27 17:03:43 -05001625 node_set(task_node(current), grp->active_nodes);
1626
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001627 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
Rik van Rielff1df892014-01-27 17:03:41 -05001628 grp->faults[i] = p->numa_faults_memory[i];
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001629
Mel Gorman989348b2013-10-07 11:29:40 +01001630 grp->total_faults = p->total_numa_faults;
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001631
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001632 list_add(&p->numa_entry, &grp->task_list);
1633 grp->nr_tasks++;
1634 rcu_assign_pointer(p->numa_group, grp);
1635 }
1636
1637 rcu_read_lock();
1638 tsk = ACCESS_ONCE(cpu_rq(cpu)->curr);
1639
1640 if (!cpupid_match_pid(tsk, cpupid))
Peter Zijlstra33547812013-10-09 10:24:48 +02001641 goto no_join;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001642
1643 grp = rcu_dereference(tsk->numa_group);
1644 if (!grp)
Peter Zijlstra33547812013-10-09 10:24:48 +02001645 goto no_join;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001646
1647 my_grp = p->numa_group;
1648 if (grp == my_grp)
Peter Zijlstra33547812013-10-09 10:24:48 +02001649 goto no_join;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001650
1651 /*
1652 * Only join the other group if its bigger; if we're the bigger group,
1653 * the other task will join us.
1654 */
1655 if (my_grp->nr_tasks > grp->nr_tasks)
Peter Zijlstra33547812013-10-09 10:24:48 +02001656 goto no_join;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001657
1658 /*
1659 * Tie-break on the grp address.
1660 */
1661 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
Peter Zijlstra33547812013-10-09 10:24:48 +02001662 goto no_join;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001663
Rik van Rieldabe1d92013-10-07 11:29:34 +01001664 /* Always join threads in the same process. */
1665 if (tsk->mm == current->mm)
1666 join = true;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001667
Rik van Rieldabe1d92013-10-07 11:29:34 +01001668 /* Simple filter to avoid false positives due to PID collisions */
1669 if (flags & TNF_SHARED)
1670 join = true;
1671
Mel Gorman3e6a9412013-10-07 11:29:35 +01001672 /* Update priv based on whether false sharing was detected */
1673 *priv = !join;
1674
Rik van Rieldabe1d92013-10-07 11:29:34 +01001675 if (join && !get_numa_group(grp))
Peter Zijlstra33547812013-10-09 10:24:48 +02001676 goto no_join;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001677
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001678 rcu_read_unlock();
1679
1680 if (!join)
1681 return;
1682
Mike Galbraith60e69ee2014-04-07 10:55:15 +02001683 BUG_ON(irqs_disabled());
1684 double_lock_irq(&my_grp->lock, &grp->lock);
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001685
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001686 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
Rik van Rielff1df892014-01-27 17:03:41 -05001687 my_grp->faults[i] -= p->numa_faults_memory[i];
1688 grp->faults[i] += p->numa_faults_memory[i];
Mel Gorman989348b2013-10-07 11:29:40 +01001689 }
1690 my_grp->total_faults -= p->total_numa_faults;
1691 grp->total_faults += p->total_numa_faults;
1692
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001693 list_move(&p->numa_entry, &grp->task_list);
1694 my_grp->nr_tasks--;
1695 grp->nr_tasks++;
1696
1697 spin_unlock(&my_grp->lock);
Mike Galbraith60e69ee2014-04-07 10:55:15 +02001698 spin_unlock_irq(&grp->lock);
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001699
1700 rcu_assign_pointer(p->numa_group, grp);
1701
1702 put_numa_group(my_grp);
Peter Zijlstra33547812013-10-09 10:24:48 +02001703 return;
1704
1705no_join:
1706 rcu_read_unlock();
1707 return;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001708}
1709
1710void task_numa_free(struct task_struct *p)
1711{
1712 struct numa_group *grp = p->numa_group;
1713 int i;
Rik van Rielff1df892014-01-27 17:03:41 -05001714 void *numa_faults = p->numa_faults_memory;
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001715
1716 if (grp) {
Mike Galbraith60e69ee2014-04-07 10:55:15 +02001717 spin_lock_irq(&grp->lock);
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001718 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
Rik van Rielff1df892014-01-27 17:03:41 -05001719 grp->faults[i] -= p->numa_faults_memory[i];
Mel Gorman989348b2013-10-07 11:29:40 +01001720 grp->total_faults -= p->total_numa_faults;
1721
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001722 list_del(&p->numa_entry);
1723 grp->nr_tasks--;
Mike Galbraith60e69ee2014-04-07 10:55:15 +02001724 spin_unlock_irq(&grp->lock);
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001725 rcu_assign_pointer(p->numa_group, NULL);
1726 put_numa_group(grp);
1727 }
1728
Rik van Rielff1df892014-01-27 17:03:41 -05001729 p->numa_faults_memory = NULL;
1730 p->numa_faults_buffer_memory = NULL;
Rik van Riel50ec8a42014-01-27 17:03:42 -05001731 p->numa_faults_cpu= NULL;
1732 p->numa_faults_buffer_cpu = NULL;
Rik van Riel82727012013-10-07 11:29:28 +01001733 kfree(numa_faults);
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001734}
1735
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001736/*
1737 * Got a PROT_NONE fault for a page on @node.
1738 */
Rik van Riel58b46da2014-01-27 17:03:47 -05001739void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001740{
1741 struct task_struct *p = current;
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001742 bool migrated = flags & TNF_MIGRATED;
Rik van Riel58b46da2014-01-27 17:03:47 -05001743 int cpu_node = task_node(current);
Rik van Riel792568e2014-04-11 13:00:27 -04001744 int local = !!(flags & TNF_FAULT_LOCAL);
Mel Gormanac8e8952013-10-07 11:29:03 +01001745 int priv;
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001746
Dave Kleikamp10e84b92013-07-31 13:53:35 -07001747 if (!numabalancing_enabled)
Mel Gorman1a687c22012-11-22 11:16:36 +00001748 return;
1749
Mel Gorman9ff1d9f2013-10-07 11:29:04 +01001750 /* for example, ksmd faulting in a user's mm */
1751 if (!p->mm)
1752 return;
1753
Rik van Riel82727012013-10-07 11:29:28 +01001754 /* Do not worry about placement if exiting */
1755 if (p->state == TASK_DEAD)
1756 return;
1757
Mel Gormanf809ca92013-10-07 11:28:57 +01001758 /* Allocate buffer to track faults on a per-node basis */
Rik van Rielff1df892014-01-27 17:03:41 -05001759 if (unlikely(!p->numa_faults_memory)) {
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001760 int size = sizeof(*p->numa_faults_memory) *
1761 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
Mel Gormanf809ca92013-10-07 11:28:57 +01001762
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001763 p->numa_faults_memory = kzalloc(size, GFP_KERNEL|__GFP_NOWARN);
Rik van Rielff1df892014-01-27 17:03:41 -05001764 if (!p->numa_faults_memory)
Mel Gormanf809ca92013-10-07 11:28:57 +01001765 return;
Mel Gorman745d6142013-10-07 11:28:59 +01001766
Rik van Rielff1df892014-01-27 17:03:41 -05001767 BUG_ON(p->numa_faults_buffer_memory);
Rik van Rielbe1e4e72014-01-27 17:03:48 -05001768 /*
1769 * The averaged statistics, shared & private, memory & cpu,
1770 * occupy the first half of the array. The second half of the
1771 * array is for current counters, which are averaged into the
1772 * first set by task_numa_placement.
1773 */
Rik van Riel50ec8a42014-01-27 17:03:42 -05001774 p->numa_faults_cpu = p->numa_faults_memory + (2 * nr_node_ids);
1775 p->numa_faults_buffer_memory = p->numa_faults_memory + (4 * nr_node_ids);
1776 p->numa_faults_buffer_cpu = p->numa_faults_memory + (6 * nr_node_ids);
Mel Gorman83e1d2c2013-10-07 11:29:27 +01001777 p->total_numa_faults = 0;
Rik van Riel04bb2f92013-10-07 11:29:36 +01001778 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
Mel Gormanf809ca92013-10-07 11:28:57 +01001779 }
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001780
Mel Gormanfb003b82012-11-15 09:01:14 +00001781 /*
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001782 * First accesses are treated as private, otherwise consider accesses
1783 * to be private if the accessing pid has not changed
1784 */
1785 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
1786 priv = 1;
1787 } else {
1788 priv = cpupid_match_pid(p, last_cpupid);
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001789 if (!priv && !(flags & TNF_NO_GROUP))
Mel Gorman3e6a9412013-10-07 11:29:35 +01001790 task_numa_group(p, last_cpupid, flags, &priv);
Peter Zijlstra8c8a7432013-10-07 11:29:21 +01001791 }
1792
Rik van Riel792568e2014-04-11 13:00:27 -04001793 /*
1794 * If a workload spans multiple NUMA nodes, a shared fault that
1795 * occurs wholly within the set of nodes that the workload is
1796 * actively using should be counted as local. This allows the
1797 * scan rate to slow down when a workload has settled down.
1798 */
1799 if (!priv && !local && p->numa_group &&
1800 node_isset(cpu_node, p->numa_group->active_nodes) &&
1801 node_isset(mem_node, p->numa_group->active_nodes))
1802 local = 1;
1803
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001804 task_numa_placement(p);
Mel Gormanf809ca92013-10-07 11:28:57 +01001805
Rik van Riel2739d3e2013-10-07 11:29:41 +01001806 /*
1807 * Retry task to preferred node migration periodically, in case it
1808 * case it previously failed, or the scheduler moved us.
1809 */
1810 if (time_after(jiffies, p->numa_migrate_retry))
Mel Gorman6b9a7462013-10-07 11:29:11 +01001811 numa_migrate_preferred(p);
1812
Ingo Molnarb32e86b2013-10-07 11:29:30 +01001813 if (migrated)
1814 p->numa_pages_migrated += pages;
1815
Rik van Riel58b46da2014-01-27 17:03:47 -05001816 p->numa_faults_buffer_memory[task_faults_idx(mem_node, priv)] += pages;
1817 p->numa_faults_buffer_cpu[task_faults_idx(cpu_node, priv)] += pages;
Rik van Riel792568e2014-04-11 13:00:27 -04001818 p->numa_faults_locality[local] += pages;
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001819}
1820
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001821static void reset_ptenuma_scan(struct task_struct *p)
1822{
1823 ACCESS_ONCE(p->mm->numa_scan_seq)++;
1824 p->mm->numa_scan_offset = 0;
1825}
1826
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001827/*
1828 * The expensive part of numa migration is done from task_work context.
1829 * Triggered from task_tick_numa().
1830 */
1831void task_numa_work(struct callback_head *work)
1832{
1833 unsigned long migrate, next_scan, now = jiffies;
1834 struct task_struct *p = current;
1835 struct mm_struct *mm = p->mm;
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001836 struct vm_area_struct *vma;
Mel Gorman9f406042012-11-14 18:34:32 +00001837 unsigned long start, end;
Mel Gorman598f0ec2013-10-07 11:28:55 +01001838 unsigned long nr_pte_updates = 0;
Mel Gorman9f406042012-11-14 18:34:32 +00001839 long pages;
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001840
1841 WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
1842
1843 work->next = work; /* protect against double add */
1844 /*
1845 * Who cares about NUMA placement when they're dying.
1846 *
1847 * NOTE: make sure not to dereference p->mm before this check,
1848 * exit_task_work() happens _after_ exit_mm() so we could be called
1849 * without p->mm even though we still had it when we enqueued this
1850 * work.
1851 */
1852 if (p->flags & PF_EXITING)
1853 return;
1854
Mel Gorman930aa172013-10-07 11:29:37 +01001855 if (!mm->numa_next_scan) {
Mel Gorman7e8d16b2013-10-07 11:28:54 +01001856 mm->numa_next_scan = now +
1857 msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
Mel Gormanb8593bf2012-11-21 01:18:23 +00001858 }
1859
1860 /*
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001861 * Enforce maximal scan/migration frequency..
1862 */
1863 migrate = mm->numa_next_scan;
1864 if (time_before(now, migrate))
1865 return;
1866
Mel Gorman598f0ec2013-10-07 11:28:55 +01001867 if (p->numa_scan_period == 0) {
1868 p->numa_scan_period_max = task_scan_max(p);
1869 p->numa_scan_period = task_scan_min(p);
1870 }
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001871
Mel Gormanfb003b82012-11-15 09:01:14 +00001872 next_scan = now + msecs_to_jiffies(p->numa_scan_period);
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001873 if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
1874 return;
1875
Mel Gormane14808b2012-11-19 10:59:15 +00001876 /*
Peter Zijlstra19a78d12013-10-07 11:28:51 +01001877 * Delay this task enough that another task of this mm will likely win
1878 * the next time around.
1879 */
1880 p->node_stamp += 2 * TICK_NSEC;
1881
Mel Gorman9f406042012-11-14 18:34:32 +00001882 start = mm->numa_scan_offset;
1883 pages = sysctl_numa_balancing_scan_size;
1884 pages <<= 20 - PAGE_SHIFT; /* MB in pages */
1885 if (!pages)
1886 return;
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001887
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001888 down_read(&mm->mmap_sem);
Mel Gorman9f406042012-11-14 18:34:32 +00001889 vma = find_vma(mm, start);
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001890 if (!vma) {
1891 reset_ptenuma_scan(p);
Mel Gorman9f406042012-11-14 18:34:32 +00001892 start = 0;
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001893 vma = mm->mmap;
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001894 }
Mel Gorman9f406042012-11-14 18:34:32 +00001895 for (; vma; vma = vma->vm_next) {
Mel Gormanfc3147242013-10-07 11:29:09 +01001896 if (!vma_migratable(vma) || !vma_policy_mof(p, vma))
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001897 continue;
1898
Mel Gorman4591ce4f2013-10-07 11:29:13 +01001899 /*
1900 * Shared library pages mapped by multiple processes are not
1901 * migrated as it is expected they are cache replicated. Avoid
1902 * hinting faults in read-only file-backed mappings or the vdso
1903 * as migrating the pages will be of marginal benefit.
1904 */
1905 if (!vma->vm_mm ||
1906 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
1907 continue;
1908
Mel Gorman3c67f472013-12-18 17:08:40 -08001909 /*
1910 * Skip inaccessible VMAs to avoid any confusion between
1911 * PROT_NONE and NUMA hinting ptes
1912 */
1913 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
1914 continue;
1915
Mel Gorman9f406042012-11-14 18:34:32 +00001916 do {
1917 start = max(start, vma->vm_start);
1918 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
1919 end = min(end, vma->vm_end);
Mel Gorman598f0ec2013-10-07 11:28:55 +01001920 nr_pte_updates += change_prot_numa(vma, start, end);
1921
1922 /*
1923 * Scan sysctl_numa_balancing_scan_size but ensure that
1924 * at least one PTE is updated so that unused virtual
1925 * address space is quickly skipped.
1926 */
1927 if (nr_pte_updates)
1928 pages -= (end - start) >> PAGE_SHIFT;
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001929
Mel Gorman9f406042012-11-14 18:34:32 +00001930 start = end;
1931 if (pages <= 0)
1932 goto out;
Rik van Riel3cf19622014-02-18 17:12:44 -05001933
1934 cond_resched();
Mel Gorman9f406042012-11-14 18:34:32 +00001935 } while (end != vma->vm_end);
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001936 }
1937
Mel Gorman9f406042012-11-14 18:34:32 +00001938out:
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001939 /*
Peter Zijlstrac69307d2013-10-07 11:28:41 +01001940 * It is possible to reach the end of the VMA list but the last few
1941 * VMAs are not guaranteed to the vma_migratable. If they are not, we
1942 * would find the !migratable VMA on the next scan but not reset the
1943 * scanner to the start so check it now.
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001944 */
1945 if (vma)
Mel Gorman9f406042012-11-14 18:34:32 +00001946 mm->numa_scan_offset = start;
Peter Zijlstra6e5fb222012-10-25 14:16:45 +02001947 else
1948 reset_ptenuma_scan(p);
1949 up_read(&mm->mmap_sem);
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001950}
1951
1952/*
1953 * Drive the periodic memory faults..
1954 */
1955void task_tick_numa(struct rq *rq, struct task_struct *curr)
1956{
1957 struct callback_head *work = &curr->numa_work;
1958 u64 period, now;
1959
1960 /*
1961 * We don't care about NUMA placement if we don't have memory.
1962 */
1963 if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work)
1964 return;
1965
1966 /*
1967 * Using runtime rather than walltime has the dual advantage that
1968 * we (mostly) drive the selection from busy threads and that the
1969 * task needs to have done some actual work before we bother with
1970 * NUMA placement.
1971 */
1972 now = curr->se.sum_exec_runtime;
1973 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
1974
1975 if (now - curr->node_stamp > period) {
Peter Zijlstra4b96a292012-10-25 14:16:47 +02001976 if (!curr->node_stamp)
Mel Gorman598f0ec2013-10-07 11:28:55 +01001977 curr->numa_scan_period = task_scan_min(curr);
Peter Zijlstra19a78d12013-10-07 11:28:51 +01001978 curr->node_stamp += period;
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001979
1980 if (!time_before(jiffies, curr->mm->numa_next_scan)) {
1981 init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */
1982 task_work_add(curr, work, true);
1983 }
1984 }
1985}
1986#else
1987static void task_tick_numa(struct rq *rq, struct task_struct *curr)
1988{
1989}
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01001990
1991static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
1992{
1993}
1994
1995static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
1996{
1997}
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001998#endif /* CONFIG_NUMA_BALANCING */
1999
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002000static void
2001account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2002{
2003 update_load_add(&cfs_rq->load, se->load.weight);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02002004 if (!parent_entity(se))
Peter Zijlstra029632f2011-10-25 10:00:11 +02002005 update_load_add(&rq_of(cfs_rq)->load, se->load.weight);
Peter Zijlstra367456c2012-02-20 21:49:09 +01002006#ifdef CONFIG_SMP
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01002007 if (entity_is_task(se)) {
2008 struct rq *rq = rq_of(cfs_rq);
2009
2010 account_numa_enqueue(rq, task_of(se));
2011 list_add(&se->group_node, &rq->cfs_tasks);
2012 }
Peter Zijlstra367456c2012-02-20 21:49:09 +01002013#endif
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002014 cfs_rq->nr_running++;
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002015}
2016
2017static void
2018account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2019{
2020 update_load_sub(&cfs_rq->load, se->load.weight);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02002021 if (!parent_entity(se))
Peter Zijlstra029632f2011-10-25 10:00:11 +02002022 update_load_sub(&rq_of(cfs_rq)->load, se->load.weight);
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01002023 if (entity_is_task(se)) {
2024 account_numa_dequeue(rq_of(cfs_rq), task_of(se));
Bharata B Raob87f1722008-09-25 09:53:54 +05302025 list_del_init(&se->group_node);
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01002026 }
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002027 cfs_rq->nr_running--;
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002028}
2029
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002030#ifdef CONFIG_FAIR_GROUP_SCHED
2031# ifdef CONFIG_SMP
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02002032static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq)
2033{
2034 long tg_weight;
2035
2036 /*
2037 * Use this CPU's actual weight instead of the last load_contribution
2038 * to gain a more accurate current total weight. See
2039 * update_cfs_rq_load_contribution().
2040 */
Alex Shibf5b9862013-06-20 10:18:54 +08002041 tg_weight = atomic_long_read(&tg->load_avg);
Paul Turner82958362012-10-04 13:18:31 +02002042 tg_weight -= cfs_rq->tg_load_contrib;
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02002043 tg_weight += cfs_rq->load.weight;
2044
2045 return tg_weight;
2046}
2047
Paul Turner6d5ab292011-01-21 20:45:01 -08002048static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002049{
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02002050 long tg_weight, load, shares;
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002051
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02002052 tg_weight = calc_tg_weight(tg, cfs_rq);
Paul Turner6d5ab292011-01-21 20:45:01 -08002053 load = cfs_rq->load.weight;
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002054
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002055 shares = (tg->shares * load);
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02002056 if (tg_weight)
2057 shares /= tg_weight;
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002058
2059 if (shares < MIN_SHARES)
2060 shares = MIN_SHARES;
2061 if (shares > tg->shares)
2062 shares = tg->shares;
2063
2064 return shares;
2065}
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002066# else /* CONFIG_SMP */
Paul Turner6d5ab292011-01-21 20:45:01 -08002067static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002068{
2069 return tg->shares;
2070}
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002071# endif /* CONFIG_SMP */
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002072static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
2073 unsigned long weight)
2074{
Paul Turner19e5eeb2010-12-15 19:10:18 -08002075 if (se->on_rq) {
2076 /* commit outstanding execution time */
2077 if (cfs_rq->curr == se)
2078 update_curr(cfs_rq);
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002079 account_entity_dequeue(cfs_rq, se);
Paul Turner19e5eeb2010-12-15 19:10:18 -08002080 }
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002081
2082 update_load_set(&se->load, weight);
2083
2084 if (se->on_rq)
2085 account_entity_enqueue(cfs_rq, se);
2086}
2087
Paul Turner82958362012-10-04 13:18:31 +02002088static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
2089
Paul Turner6d5ab292011-01-21 20:45:01 -08002090static void update_cfs_shares(struct cfs_rq *cfs_rq)
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002091{
2092 struct task_group *tg;
2093 struct sched_entity *se;
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002094 long shares;
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002095
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002096 tg = cfs_rq->tg;
2097 se = tg->se[cpu_of(rq_of(cfs_rq))];
Paul Turner64660c82011-07-21 09:43:36 -07002098 if (!se || throttled_hierarchy(cfs_rq))
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002099 return;
Yong Zhang3ff6dca2011-01-24 15:33:52 +08002100#ifndef CONFIG_SMP
2101 if (likely(se->load.weight == tg->shares))
2102 return;
2103#endif
Paul Turner6d5ab292011-01-21 20:45:01 -08002104 shares = calc_cfs_shares(cfs_rq, tg);
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002105
2106 reweight_entity(cfs_rq_of(se), se, shares);
2107}
2108#else /* CONFIG_FAIR_GROUP_SCHED */
Paul Turner6d5ab292011-01-21 20:45:01 -08002109static inline void update_cfs_shares(struct cfs_rq *cfs_rq)
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002110{
2111}
2112#endif /* CONFIG_FAIR_GROUP_SCHED */
2113
Alex Shi141965c2013-06-26 13:05:39 +08002114#ifdef CONFIG_SMP
Paul Turner9d85f212012-10-04 13:18:29 +02002115/*
Paul Turner5b51f2f2012-10-04 13:18:32 +02002116 * We choose a half-life close to 1 scheduling period.
2117 * Note: The tables below are dependent on this value.
2118 */
2119#define LOAD_AVG_PERIOD 32
2120#define LOAD_AVG_MAX 47742 /* maximum possible load avg */
2121#define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_MAX_AVG */
2122
2123/* Precomputed fixed inverse multiplies for multiplication by y^n */
2124static const u32 runnable_avg_yN_inv[] = {
2125 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6,
2126 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85,
2127 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581,
2128 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9,
2129 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80,
2130 0x85aac367, 0x82cd8698,
2131};
2132
2133/*
2134 * Precomputed \Sum y^k { 1<=k<=n }. These are floor(true_value) to prevent
2135 * over-estimates when re-combining.
2136 */
2137static const u32 runnable_avg_yN_sum[] = {
2138 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103,
2139 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082,
2140 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371,
2141};
2142
2143/*
Paul Turner9d85f212012-10-04 13:18:29 +02002144 * Approximate:
2145 * val * y^n, where y^32 ~= 0.5 (~1 scheduling period)
2146 */
2147static __always_inline u64 decay_load(u64 val, u64 n)
2148{
Paul Turner5b51f2f2012-10-04 13:18:32 +02002149 unsigned int local_n;
2150
2151 if (!n)
2152 return val;
2153 else if (unlikely(n > LOAD_AVG_PERIOD * 63))
2154 return 0;
2155
2156 /* after bounds checking we can collapse to 32-bit */
2157 local_n = n;
2158
2159 /*
2160 * As y^PERIOD = 1/2, we can combine
2161 * y^n = 1/2^(n/PERIOD) * k^(n%PERIOD)
2162 * With a look-up table which covers k^n (n<PERIOD)
2163 *
2164 * To achieve constant time decay_load.
2165 */
2166 if (unlikely(local_n >= LOAD_AVG_PERIOD)) {
2167 val >>= local_n / LOAD_AVG_PERIOD;
2168 local_n %= LOAD_AVG_PERIOD;
Paul Turner9d85f212012-10-04 13:18:29 +02002169 }
2170
Paul Turner5b51f2f2012-10-04 13:18:32 +02002171 val *= runnable_avg_yN_inv[local_n];
2172 /* We don't use SRR here since we always want to round down. */
2173 return val >> 32;
2174}
2175
2176/*
2177 * For updates fully spanning n periods, the contribution to runnable
2178 * average will be: \Sum 1024*y^n
2179 *
2180 * We can compute this reasonably efficiently by combining:
2181 * y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for n <PERIOD}
2182 */
2183static u32 __compute_runnable_contrib(u64 n)
2184{
2185 u32 contrib = 0;
2186
2187 if (likely(n <= LOAD_AVG_PERIOD))
2188 return runnable_avg_yN_sum[n];
2189 else if (unlikely(n >= LOAD_AVG_MAX_N))
2190 return LOAD_AVG_MAX;
2191
2192 /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */
2193 do {
2194 contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */
2195 contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD];
2196
2197 n -= LOAD_AVG_PERIOD;
2198 } while (n > LOAD_AVG_PERIOD);
2199
2200 contrib = decay_load(contrib, n);
2201 return contrib + runnable_avg_yN_sum[n];
Paul Turner9d85f212012-10-04 13:18:29 +02002202}
2203
2204/*
2205 * We can represent the historical contribution to runnable average as the
2206 * coefficients of a geometric series. To do this we sub-divide our runnable
2207 * history into segments of approximately 1ms (1024us); label the segment that
2208 * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g.
2209 *
2210 * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ...
2211 * p0 p1 p2
2212 * (now) (~1ms ago) (~2ms ago)
2213 *
2214 * Let u_i denote the fraction of p_i that the entity was runnable.
2215 *
2216 * We then designate the fractions u_i as our co-efficients, yielding the
2217 * following representation of historical load:
2218 * u_0 + u_1*y + u_2*y^2 + u_3*y^3 + ...
2219 *
2220 * We choose y based on the with of a reasonably scheduling period, fixing:
2221 * y^32 = 0.5
2222 *
2223 * This means that the contribution to load ~32ms ago (u_32) will be weighted
2224 * approximately half as much as the contribution to load within the last ms
2225 * (u_0).
2226 *
2227 * When a period "rolls over" and we have new u_0`, multiplying the previous
2228 * sum again by y is sufficient to update:
2229 * load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... )
2230 * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}]
2231 */
2232static __always_inline int __update_entity_runnable_avg(u64 now,
2233 struct sched_avg *sa,
2234 int runnable)
2235{
Paul Turner5b51f2f2012-10-04 13:18:32 +02002236 u64 delta, periods;
2237 u32 runnable_contrib;
Paul Turner9d85f212012-10-04 13:18:29 +02002238 int delta_w, decayed = 0;
2239
2240 delta = now - sa->last_runnable_update;
2241 /*
2242 * This should only happen when time goes backwards, which it
2243 * unfortunately does during sched clock init when we swap over to TSC.
2244 */
2245 if ((s64)delta < 0) {
2246 sa->last_runnable_update = now;
2247 return 0;
2248 }
2249
2250 /*
2251 * Use 1024ns as the unit of measurement since it's a reasonable
2252 * approximation of 1us and fast to compute.
2253 */
2254 delta >>= 10;
2255 if (!delta)
2256 return 0;
2257 sa->last_runnable_update = now;
2258
2259 /* delta_w is the amount already accumulated against our next period */
2260 delta_w = sa->runnable_avg_period % 1024;
2261 if (delta + delta_w >= 1024) {
2262 /* period roll-over */
2263 decayed = 1;
2264
2265 /*
2266 * Now that we know we're crossing a period boundary, figure
2267 * out how much from delta we need to complete the current
2268 * period and accrue it.
2269 */
2270 delta_w = 1024 - delta_w;
Paul Turner5b51f2f2012-10-04 13:18:32 +02002271 if (runnable)
2272 sa->runnable_avg_sum += delta_w;
2273 sa->runnable_avg_period += delta_w;
Paul Turner9d85f212012-10-04 13:18:29 +02002274
Paul Turner5b51f2f2012-10-04 13:18:32 +02002275 delta -= delta_w;
Paul Turner9d85f212012-10-04 13:18:29 +02002276
Paul Turner5b51f2f2012-10-04 13:18:32 +02002277 /* Figure out how many additional periods this update spans */
2278 periods = delta / 1024;
2279 delta %= 1024;
2280
2281 sa->runnable_avg_sum = decay_load(sa->runnable_avg_sum,
2282 periods + 1);
2283 sa->runnable_avg_period = decay_load(sa->runnable_avg_period,
2284 periods + 1);
2285
2286 /* Efficiently calculate \sum (1..n_period) 1024*y^i */
2287 runnable_contrib = __compute_runnable_contrib(periods);
2288 if (runnable)
2289 sa->runnable_avg_sum += runnable_contrib;
2290 sa->runnable_avg_period += runnable_contrib;
Paul Turner9d85f212012-10-04 13:18:29 +02002291 }
2292
2293 /* Remainder of delta accrued against u_0` */
2294 if (runnable)
2295 sa->runnable_avg_sum += delta;
2296 sa->runnable_avg_period += delta;
2297
2298 return decayed;
2299}
2300
Paul Turner9ee474f2012-10-04 13:18:30 +02002301/* Synchronize an entity's decay with its parenting cfs_rq.*/
Paul Turneraff3e492012-10-04 13:18:30 +02002302static inline u64 __synchronize_entity_decay(struct sched_entity *se)
Paul Turner9ee474f2012-10-04 13:18:30 +02002303{
2304 struct cfs_rq *cfs_rq = cfs_rq_of(se);
2305 u64 decays = atomic64_read(&cfs_rq->decay_counter);
2306
2307 decays -= se->avg.decay_count;
2308 if (!decays)
Paul Turneraff3e492012-10-04 13:18:30 +02002309 return 0;
Paul Turner9ee474f2012-10-04 13:18:30 +02002310
2311 se->avg.load_avg_contrib = decay_load(se->avg.load_avg_contrib, decays);
2312 se->avg.decay_count = 0;
Paul Turneraff3e492012-10-04 13:18:30 +02002313
2314 return decays;
Paul Turner9ee474f2012-10-04 13:18:30 +02002315}
2316
Paul Turnerc566e8e2012-10-04 13:18:30 +02002317#ifdef CONFIG_FAIR_GROUP_SCHED
2318static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq,
2319 int force_update)
2320{
2321 struct task_group *tg = cfs_rq->tg;
Alex Shibf5b9862013-06-20 10:18:54 +08002322 long tg_contrib;
Paul Turnerc566e8e2012-10-04 13:18:30 +02002323
2324 tg_contrib = cfs_rq->runnable_load_avg + cfs_rq->blocked_load_avg;
2325 tg_contrib -= cfs_rq->tg_load_contrib;
2326
Alex Shibf5b9862013-06-20 10:18:54 +08002327 if (force_update || abs(tg_contrib) > cfs_rq->tg_load_contrib / 8) {
2328 atomic_long_add(tg_contrib, &tg->load_avg);
Paul Turnerc566e8e2012-10-04 13:18:30 +02002329 cfs_rq->tg_load_contrib += tg_contrib;
2330 }
2331}
Paul Turner8165e142012-10-04 13:18:31 +02002332
Paul Turnerbb17f652012-10-04 13:18:31 +02002333/*
2334 * Aggregate cfs_rq runnable averages into an equivalent task_group
2335 * representation for computing load contributions.
2336 */
2337static inline void __update_tg_runnable_avg(struct sched_avg *sa,
2338 struct cfs_rq *cfs_rq)
2339{
2340 struct task_group *tg = cfs_rq->tg;
2341 long contrib;
2342
2343 /* The fraction of a cpu used by this cfs_rq */
Michal Nazarewicz85b088e2013-11-10 20:42:01 +01002344 contrib = div_u64((u64)sa->runnable_avg_sum << NICE_0_SHIFT,
Paul Turnerbb17f652012-10-04 13:18:31 +02002345 sa->runnable_avg_period + 1);
2346 contrib -= cfs_rq->tg_runnable_contrib;
2347
2348 if (abs(contrib) > cfs_rq->tg_runnable_contrib / 64) {
2349 atomic_add(contrib, &tg->runnable_avg);
2350 cfs_rq->tg_runnable_contrib += contrib;
2351 }
2352}
2353
Paul Turner8165e142012-10-04 13:18:31 +02002354static inline void __update_group_entity_contrib(struct sched_entity *se)
2355{
2356 struct cfs_rq *cfs_rq = group_cfs_rq(se);
2357 struct task_group *tg = cfs_rq->tg;
Paul Turnerbb17f652012-10-04 13:18:31 +02002358 int runnable_avg;
2359
Paul Turner8165e142012-10-04 13:18:31 +02002360 u64 contrib;
2361
2362 contrib = cfs_rq->tg_load_contrib * tg->shares;
Alex Shibf5b9862013-06-20 10:18:54 +08002363 se->avg.load_avg_contrib = div_u64(contrib,
2364 atomic_long_read(&tg->load_avg) + 1);
Paul Turnerbb17f652012-10-04 13:18:31 +02002365
2366 /*
2367 * For group entities we need to compute a correction term in the case
2368 * that they are consuming <1 cpu so that we would contribute the same
2369 * load as a task of equal weight.
2370 *
2371 * Explicitly co-ordinating this measurement would be expensive, but
2372 * fortunately the sum of each cpus contribution forms a usable
2373 * lower-bound on the true value.
2374 *
2375 * Consider the aggregate of 2 contributions. Either they are disjoint
2376 * (and the sum represents true value) or they are disjoint and we are
2377 * understating by the aggregate of their overlap.
2378 *
2379 * Extending this to N cpus, for a given overlap, the maximum amount we
2380 * understand is then n_i(n_i+1)/2 * w_i where n_i is the number of
2381 * cpus that overlap for this interval and w_i is the interval width.
2382 *
2383 * On a small machine; the first term is well-bounded which bounds the
2384 * total error since w_i is a subset of the period. Whereas on a
2385 * larger machine, while this first term can be larger, if w_i is the
2386 * of consequential size guaranteed to see n_i*w_i quickly converge to
2387 * our upper bound of 1-cpu.
2388 */
2389 runnable_avg = atomic_read(&tg->runnable_avg);
2390 if (runnable_avg < NICE_0_LOAD) {
2391 se->avg.load_avg_contrib *= runnable_avg;
2392 se->avg.load_avg_contrib >>= NICE_0_SHIFT;
2393 }
Paul Turner8165e142012-10-04 13:18:31 +02002394}
Dietmar Eggemannf5f97392014-02-26 11:19:33 +00002395
2396static inline void update_rq_runnable_avg(struct rq *rq, int runnable)
2397{
2398 __update_entity_runnable_avg(rq_clock_task(rq), &rq->avg, runnable);
2399 __update_tg_runnable_avg(&rq->avg, &rq->cfs);
2400}
Peter Zijlstra6e831252014-02-11 16:11:48 +01002401#else /* CONFIG_FAIR_GROUP_SCHED */
Paul Turnerc566e8e2012-10-04 13:18:30 +02002402static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq,
2403 int force_update) {}
Paul Turnerbb17f652012-10-04 13:18:31 +02002404static inline void __update_tg_runnable_avg(struct sched_avg *sa,
2405 struct cfs_rq *cfs_rq) {}
Paul Turner8165e142012-10-04 13:18:31 +02002406static inline void __update_group_entity_contrib(struct sched_entity *se) {}
Dietmar Eggemannf5f97392014-02-26 11:19:33 +00002407static inline void update_rq_runnable_avg(struct rq *rq, int runnable) {}
Peter Zijlstra6e831252014-02-11 16:11:48 +01002408#endif /* CONFIG_FAIR_GROUP_SCHED */
Paul Turnerc566e8e2012-10-04 13:18:30 +02002409
Paul Turner8165e142012-10-04 13:18:31 +02002410static inline void __update_task_entity_contrib(struct sched_entity *se)
2411{
2412 u32 contrib;
2413
2414 /* avoid overflowing a 32-bit type w/ SCHED_LOAD_SCALE */
2415 contrib = se->avg.runnable_avg_sum * scale_load_down(se->load.weight);
2416 contrib /= (se->avg.runnable_avg_period + 1);
2417 se->avg.load_avg_contrib = scale_load(contrib);
2418}
2419
Paul Turner2dac7542012-10-04 13:18:30 +02002420/* Compute the current contribution to load_avg by se, return any delta */
2421static long __update_entity_load_avg_contrib(struct sched_entity *se)
2422{
2423 long old_contrib = se->avg.load_avg_contrib;
2424
Paul Turner8165e142012-10-04 13:18:31 +02002425 if (entity_is_task(se)) {
2426 __update_task_entity_contrib(se);
2427 } else {
Paul Turnerbb17f652012-10-04 13:18:31 +02002428 __update_tg_runnable_avg(&se->avg, group_cfs_rq(se));
Paul Turner8165e142012-10-04 13:18:31 +02002429 __update_group_entity_contrib(se);
2430 }
Paul Turner2dac7542012-10-04 13:18:30 +02002431
2432 return se->avg.load_avg_contrib - old_contrib;
2433}
2434
Paul Turner9ee474f2012-10-04 13:18:30 +02002435static inline void subtract_blocked_load_contrib(struct cfs_rq *cfs_rq,
2436 long load_contrib)
2437{
2438 if (likely(load_contrib < cfs_rq->blocked_load_avg))
2439 cfs_rq->blocked_load_avg -= load_contrib;
2440 else
2441 cfs_rq->blocked_load_avg = 0;
2442}
2443
Paul Turnerf1b17282012-10-04 13:18:31 +02002444static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq);
2445
Paul Turner9d85f212012-10-04 13:18:29 +02002446/* Update a sched_entity's runnable average */
Paul Turner9ee474f2012-10-04 13:18:30 +02002447static inline void update_entity_load_avg(struct sched_entity *se,
2448 int update_cfs_rq)
Paul Turner9d85f212012-10-04 13:18:29 +02002449{
Paul Turner2dac7542012-10-04 13:18:30 +02002450 struct cfs_rq *cfs_rq = cfs_rq_of(se);
2451 long contrib_delta;
Paul Turnerf1b17282012-10-04 13:18:31 +02002452 u64 now;
Paul Turner2dac7542012-10-04 13:18:30 +02002453
Paul Turnerf1b17282012-10-04 13:18:31 +02002454 /*
2455 * For a group entity we need to use their owned cfs_rq_clock_task() in
2456 * case they are the parent of a throttled hierarchy.
2457 */
2458 if (entity_is_task(se))
2459 now = cfs_rq_clock_task(cfs_rq);
2460 else
2461 now = cfs_rq_clock_task(group_cfs_rq(se));
2462
2463 if (!__update_entity_runnable_avg(now, &se->avg, se->on_rq))
Paul Turner2dac7542012-10-04 13:18:30 +02002464 return;
2465
2466 contrib_delta = __update_entity_load_avg_contrib(se);
Paul Turner9ee474f2012-10-04 13:18:30 +02002467
2468 if (!update_cfs_rq)
2469 return;
2470
Paul Turner2dac7542012-10-04 13:18:30 +02002471 if (se->on_rq)
2472 cfs_rq->runnable_load_avg += contrib_delta;
Paul Turner9ee474f2012-10-04 13:18:30 +02002473 else
2474 subtract_blocked_load_contrib(cfs_rq, -contrib_delta);
2475}
2476
2477/*
2478 * Decay the load contributed by all blocked children and account this so that
2479 * their contribution may appropriately discounted when they wake up.
2480 */
Paul Turneraff3e492012-10-04 13:18:30 +02002481static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update)
Paul Turner9ee474f2012-10-04 13:18:30 +02002482{
Paul Turnerf1b17282012-10-04 13:18:31 +02002483 u64 now = cfs_rq_clock_task(cfs_rq) >> 20;
Paul Turner9ee474f2012-10-04 13:18:30 +02002484 u64 decays;
2485
2486 decays = now - cfs_rq->last_decay;
Paul Turneraff3e492012-10-04 13:18:30 +02002487 if (!decays && !force_update)
Paul Turner9ee474f2012-10-04 13:18:30 +02002488 return;
2489
Alex Shi25099402013-06-20 10:18:55 +08002490 if (atomic_long_read(&cfs_rq->removed_load)) {
2491 unsigned long removed_load;
2492 removed_load = atomic_long_xchg(&cfs_rq->removed_load, 0);
Paul Turneraff3e492012-10-04 13:18:30 +02002493 subtract_blocked_load_contrib(cfs_rq, removed_load);
2494 }
Paul Turner9ee474f2012-10-04 13:18:30 +02002495
Paul Turneraff3e492012-10-04 13:18:30 +02002496 if (decays) {
2497 cfs_rq->blocked_load_avg = decay_load(cfs_rq->blocked_load_avg,
2498 decays);
2499 atomic64_add(decays, &cfs_rq->decay_counter);
2500 cfs_rq->last_decay = now;
2501 }
Paul Turnerc566e8e2012-10-04 13:18:30 +02002502
2503 __update_cfs_rq_tg_load_contrib(cfs_rq, force_update);
Paul Turner9d85f212012-10-04 13:18:29 +02002504}
Ben Segall18bf2802012-10-04 12:51:20 +02002505
Paul Turner2dac7542012-10-04 13:18:30 +02002506/* Add the load generated by se into cfs_rq's child load-average */
2507static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq,
Paul Turner9ee474f2012-10-04 13:18:30 +02002508 struct sched_entity *se,
2509 int wakeup)
Paul Turner2dac7542012-10-04 13:18:30 +02002510{
Paul Turneraff3e492012-10-04 13:18:30 +02002511 /*
2512 * We track migrations using entity decay_count <= 0, on a wake-up
2513 * migration we use a negative decay count to track the remote decays
2514 * accumulated while sleeping.
Alex Shia75cdaa2013-06-20 10:18:47 +08002515 *
2516 * Newly forked tasks are enqueued with se->avg.decay_count == 0, they
2517 * are seen by enqueue_entity_load_avg() as a migration with an already
2518 * constructed load_avg_contrib.
Paul Turneraff3e492012-10-04 13:18:30 +02002519 */
2520 if (unlikely(se->avg.decay_count <= 0)) {
Frederic Weisbecker78becc22013-04-12 01:51:02 +02002521 se->avg.last_runnable_update = rq_clock_task(rq_of(cfs_rq));
Paul Turneraff3e492012-10-04 13:18:30 +02002522 if (se->avg.decay_count) {
2523 /*
2524 * In a wake-up migration we have to approximate the
2525 * time sleeping. This is because we can't synchronize
2526 * clock_task between the two cpus, and it is not
2527 * guaranteed to be read-safe. Instead, we can
2528 * approximate this using our carried decays, which are
2529 * explicitly atomically readable.
2530 */
2531 se->avg.last_runnable_update -= (-se->avg.decay_count)
2532 << 20;
2533 update_entity_load_avg(se, 0);
2534 /* Indicate that we're now synchronized and on-rq */
2535 se->avg.decay_count = 0;
2536 }
Paul Turner9ee474f2012-10-04 13:18:30 +02002537 wakeup = 0;
2538 } else {
Vincent Guittot93906752014-01-22 08:45:34 +01002539 __synchronize_entity_decay(se);
Paul Turner9ee474f2012-10-04 13:18:30 +02002540 }
2541
Paul Turneraff3e492012-10-04 13:18:30 +02002542 /* migrated tasks did not contribute to our blocked load */
2543 if (wakeup) {
Paul Turner9ee474f2012-10-04 13:18:30 +02002544 subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib);
Paul Turneraff3e492012-10-04 13:18:30 +02002545 update_entity_load_avg(se, 0);
2546 }
Paul Turner9ee474f2012-10-04 13:18:30 +02002547
Paul Turner2dac7542012-10-04 13:18:30 +02002548 cfs_rq->runnable_load_avg += se->avg.load_avg_contrib;
Paul Turneraff3e492012-10-04 13:18:30 +02002549 /* we force update consideration on load-balancer moves */
2550 update_cfs_rq_blocked_load(cfs_rq, !wakeup);
Paul Turner2dac7542012-10-04 13:18:30 +02002551}
2552
Paul Turner9ee474f2012-10-04 13:18:30 +02002553/*
2554 * Remove se's load from this cfs_rq child load-average, if the entity is
2555 * transitioning to a blocked state we track its projected decay using
2556 * blocked_load_avg.
2557 */
Paul Turner2dac7542012-10-04 13:18:30 +02002558static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq,
Paul Turner9ee474f2012-10-04 13:18:30 +02002559 struct sched_entity *se,
2560 int sleep)
Paul Turner2dac7542012-10-04 13:18:30 +02002561{
Paul Turner9ee474f2012-10-04 13:18:30 +02002562 update_entity_load_avg(se, 1);
Paul Turneraff3e492012-10-04 13:18:30 +02002563 /* we force update consideration on load-balancer moves */
2564 update_cfs_rq_blocked_load(cfs_rq, !sleep);
Paul Turner9ee474f2012-10-04 13:18:30 +02002565
Paul Turner2dac7542012-10-04 13:18:30 +02002566 cfs_rq->runnable_load_avg -= se->avg.load_avg_contrib;
Paul Turner9ee474f2012-10-04 13:18:30 +02002567 if (sleep) {
2568 cfs_rq->blocked_load_avg += se->avg.load_avg_contrib;
2569 se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter);
2570 } /* migrations, e.g. sleep=0 leave decay_count == 0 */
Paul Turner2dac7542012-10-04 13:18:30 +02002571}
Vincent Guittot642dbc32013-04-18 18:34:26 +02002572
2573/*
2574 * Update the rq's load with the elapsed running time before entering
2575 * idle. if the last scheduled task is not a CFS task, idle_enter will
2576 * be the only way to update the runnable statistic.
2577 */
2578void idle_enter_fair(struct rq *this_rq)
2579{
2580 update_rq_runnable_avg(this_rq, 1);
2581}
2582
2583/*
2584 * Update the rq's load with the elapsed idle time before a task is
2585 * scheduled. if the newly scheduled task is not a CFS task, idle_exit will
2586 * be the only way to update the runnable statistic.
2587 */
2588void idle_exit_fair(struct rq *this_rq)
2589{
2590 update_rq_runnable_avg(this_rq, 0);
2591}
2592
Peter Zijlstra6e831252014-02-11 16:11:48 +01002593static int idle_balance(struct rq *this_rq);
2594
Peter Zijlstra38033c32014-01-23 20:32:21 +01002595#else /* CONFIG_SMP */
2596
Paul Turner9ee474f2012-10-04 13:18:30 +02002597static inline void update_entity_load_avg(struct sched_entity *se,
2598 int update_cfs_rq) {}
Ben Segall18bf2802012-10-04 12:51:20 +02002599static inline void update_rq_runnable_avg(struct rq *rq, int runnable) {}
Paul Turner2dac7542012-10-04 13:18:30 +02002600static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq,
Paul Turner9ee474f2012-10-04 13:18:30 +02002601 struct sched_entity *se,
2602 int wakeup) {}
Paul Turner2dac7542012-10-04 13:18:30 +02002603static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq,
Paul Turner9ee474f2012-10-04 13:18:30 +02002604 struct sched_entity *se,
2605 int sleep) {}
Paul Turneraff3e492012-10-04 13:18:30 +02002606static inline void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq,
2607 int force_update) {}
Peter Zijlstra6e831252014-02-11 16:11:48 +01002608
2609static inline int idle_balance(struct rq *rq)
2610{
2611 return 0;
2612}
2613
Peter Zijlstra38033c32014-01-23 20:32:21 +01002614#endif /* CONFIG_SMP */
Paul Turner9d85f212012-10-04 13:18:29 +02002615
Ingo Molnar2396af62007-08-09 11:16:48 +02002616static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002617{
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002618#ifdef CONFIG_SCHEDSTATS
Peter Zijlstrae4143142009-07-23 20:13:26 +02002619 struct task_struct *tsk = NULL;
2620
2621 if (entity_is_task(se))
2622 tsk = task_of(se);
2623
Lucas De Marchi41acab82010-03-10 23:37:45 -03002624 if (se->statistics.sleep_start) {
Frederic Weisbecker78becc22013-04-12 01:51:02 +02002625 u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.sleep_start;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002626
2627 if ((s64)delta < 0)
2628 delta = 0;
2629
Lucas De Marchi41acab82010-03-10 23:37:45 -03002630 if (unlikely(delta > se->statistics.sleep_max))
2631 se->statistics.sleep_max = delta;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002632
Peter Zijlstra8c79a042012-01-30 14:51:37 +01002633 se->statistics.sleep_start = 0;
Lucas De Marchi41acab82010-03-10 23:37:45 -03002634 se->statistics.sum_sleep_runtime += delta;
Arjan van de Ven97455122008-01-25 21:08:34 +01002635
Peter Zijlstra768d0c22009-07-23 20:13:26 +02002636 if (tsk) {
Peter Zijlstrae4143142009-07-23 20:13:26 +02002637 account_scheduler_latency(tsk, delta >> 10, 1);
Peter Zijlstra768d0c22009-07-23 20:13:26 +02002638 trace_sched_stat_sleep(tsk, delta);
2639 }
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002640 }
Lucas De Marchi41acab82010-03-10 23:37:45 -03002641 if (se->statistics.block_start) {
Frederic Weisbecker78becc22013-04-12 01:51:02 +02002642 u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.block_start;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002643
2644 if ((s64)delta < 0)
2645 delta = 0;
2646
Lucas De Marchi41acab82010-03-10 23:37:45 -03002647 if (unlikely(delta > se->statistics.block_max))
2648 se->statistics.block_max = delta;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002649
Peter Zijlstra8c79a042012-01-30 14:51:37 +01002650 se->statistics.block_start = 0;
Lucas De Marchi41acab82010-03-10 23:37:45 -03002651 se->statistics.sum_sleep_runtime += delta;
Ingo Molnar30084fb2007-10-02 14:13:08 +02002652
Peter Zijlstrae4143142009-07-23 20:13:26 +02002653 if (tsk) {
Arjan van de Ven8f0dfc32009-07-20 11:26:58 -07002654 if (tsk->in_iowait) {
Lucas De Marchi41acab82010-03-10 23:37:45 -03002655 se->statistics.iowait_sum += delta;
2656 se->statistics.iowait_count++;
Peter Zijlstra768d0c22009-07-23 20:13:26 +02002657 trace_sched_stat_iowait(tsk, delta);
Arjan van de Ven8f0dfc32009-07-20 11:26:58 -07002658 }
2659
Andrew Vaginb781a602011-11-28 12:03:35 +03002660 trace_sched_stat_blocked(tsk, delta);
2661
Peter Zijlstrae4143142009-07-23 20:13:26 +02002662 /*
2663 * Blocking time is in units of nanosecs, so shift by
2664 * 20 to get a milliseconds-range estimation of the
2665 * amount of time that the task spent sleeping:
2666 */
2667 if (unlikely(prof_on == SLEEP_PROFILING)) {
2668 profile_hits(SLEEP_PROFILING,
2669 (void *)get_wchan(tsk),
2670 delta >> 20);
2671 }
2672 account_scheduler_latency(tsk, delta >> 10, 0);
Ingo Molnar30084fb2007-10-02 14:13:08 +02002673 }
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002674 }
2675#endif
2676}
2677
Peter Zijlstraddc97292007-10-15 17:00:10 +02002678static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
2679{
2680#ifdef CONFIG_SCHED_DEBUG
2681 s64 d = se->vruntime - cfs_rq->min_vruntime;
2682
2683 if (d < 0)
2684 d = -d;
2685
2686 if (d > 3*sysctl_sched_latency)
2687 schedstat_inc(cfs_rq, nr_spread_over);
2688#endif
2689}
2690
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002691static void
Peter Zijlstraaeb73b02007-10-15 17:00:05 +02002692place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
2693{
Peter Zijlstra1af5f732008-10-24 11:06:13 +02002694 u64 vruntime = cfs_rq->min_vruntime;
Peter Zijlstra94dfb5e2007-10-15 17:00:05 +02002695
Peter Zijlstra2cb86002007-11-09 22:39:37 +01002696 /*
2697 * The 'current' period is already promised to the current tasks,
2698 * however the extra weight of the new task will slow them down a
2699 * little, place the new task so that it fits in the slot that
2700 * stays open at the end.
2701 */
Peter Zijlstra94dfb5e2007-10-15 17:00:05 +02002702 if (initial && sched_feat(START_DEBIT))
Peter Zijlstraf9c0b092008-10-17 19:27:04 +02002703 vruntime += sched_vslice(cfs_rq, se);
Peter Zijlstraaeb73b02007-10-15 17:00:05 +02002704
Mike Galbraitha2e7a7e2009-09-18 09:19:25 +02002705 /* sleeps up to a single latency don't count. */
Mike Galbraith5ca98802010-03-11 17:17:17 +01002706 if (!initial) {
Mike Galbraitha2e7a7e2009-09-18 09:19:25 +02002707 unsigned long thresh = sysctl_sched_latency;
Peter Zijlstraa7be37a2008-06-27 13:41:11 +02002708
Mike Galbraitha2e7a7e2009-09-18 09:19:25 +02002709 /*
Mike Galbraitha2e7a7e2009-09-18 09:19:25 +02002710 * Halve their sleep time's effect, to allow
2711 * for a gentler effect of sleepers:
2712 */
2713 if (sched_feat(GENTLE_FAIR_SLEEPERS))
2714 thresh >>= 1;
Ingo Molnar51e03042009-09-16 08:54:45 +02002715
Mike Galbraitha2e7a7e2009-09-18 09:19:25 +02002716 vruntime -= thresh;
Peter Zijlstraaeb73b02007-10-15 17:00:05 +02002717 }
2718
Mike Galbraithb5d9d732009-09-08 11:12:28 +02002719 /* ensure we never gain time by being placed backwards. */
Viresh Kumar16c8f1c2012-11-08 13:33:46 +05302720 se->vruntime = max_vruntime(se->vruntime, vruntime);
Peter Zijlstraaeb73b02007-10-15 17:00:05 +02002721}
2722
Paul Turnerd3d9dc32011-07-21 09:43:39 -07002723static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
2724
Peter Zijlstraaeb73b02007-10-15 17:00:05 +02002725static void
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01002726enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002727{
2728 /*
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01002729 * Update the normalized vruntime before updating min_vruntime
Kamalesh Babulal0fc576d2013-06-27 11:24:18 +05302730 * through calling update_curr().
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01002731 */
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002732 if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING))
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01002733 se->vruntime += cfs_rq->min_vruntime;
2734
2735 /*
Dmitry Adamushkoa2a2d682007-10-15 17:00:13 +02002736 * Update run-time statistics of the 'current'.
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002737 */
Ingo Molnarb7cc0892007-08-09 11:16:47 +02002738 update_curr(cfs_rq);
Paul Turnerf269ae02012-10-04 13:18:31 +02002739 enqueue_entity_load_avg(cfs_rq, se, flags & ENQUEUE_WAKEUP);
Linus Torvalds17bc14b2012-12-14 07:20:43 -08002740 account_entity_enqueue(cfs_rq, se);
2741 update_cfs_shares(cfs_rq);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002742
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01002743 if (flags & ENQUEUE_WAKEUP) {
Peter Zijlstraaeb73b02007-10-15 17:00:05 +02002744 place_entity(cfs_rq, se, 0);
Ingo Molnar2396af62007-08-09 11:16:48 +02002745 enqueue_sleeper(cfs_rq, se);
Ingo Molnare9acbff2007-10-15 17:00:04 +02002746 }
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002747
Ingo Molnard2417e52007-08-09 11:16:47 +02002748 update_stats_enqueue(cfs_rq, se);
Peter Zijlstraddc97292007-10-15 17:00:10 +02002749 check_spread(cfs_rq, se);
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02002750 if (se != cfs_rq->curr)
2751 __enqueue_entity(cfs_rq, se);
Peter Zijlstra2069dd72010-11-15 15:47:00 -08002752 se->on_rq = 1;
Peter Zijlstra3d4b47b2010-11-15 15:47:01 -08002753
Paul Turnerd3d9dc32011-07-21 09:43:39 -07002754 if (cfs_rq->nr_running == 1) {
Peter Zijlstra3d4b47b2010-11-15 15:47:01 -08002755 list_add_leaf_cfs_rq(cfs_rq);
Paul Turnerd3d9dc32011-07-21 09:43:39 -07002756 check_enqueue_throttle(cfs_rq);
2757 }
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002758}
2759
Rik van Riel2c13c9192011-02-01 09:48:37 -05002760static void __clear_buddies_last(struct sched_entity *se)
Peter Zijlstra2002c692008-11-11 11:52:33 +01002761{
Rik van Riel2c13c9192011-02-01 09:48:37 -05002762 for_each_sched_entity(se) {
2763 struct cfs_rq *cfs_rq = cfs_rq_of(se);
Peter Zijlstraf1044792012-02-11 06:05:00 +01002764 if (cfs_rq->last != se)
Rik van Riel2c13c9192011-02-01 09:48:37 -05002765 break;
Peter Zijlstraf1044792012-02-11 06:05:00 +01002766
2767 cfs_rq->last = NULL;
Rik van Riel2c13c9192011-02-01 09:48:37 -05002768 }
2769}
Peter Zijlstra2002c692008-11-11 11:52:33 +01002770
Rik van Riel2c13c9192011-02-01 09:48:37 -05002771static void __clear_buddies_next(struct sched_entity *se)
2772{
2773 for_each_sched_entity(se) {
2774 struct cfs_rq *cfs_rq = cfs_rq_of(se);
Peter Zijlstraf1044792012-02-11 06:05:00 +01002775 if (cfs_rq->next != se)
Rik van Riel2c13c9192011-02-01 09:48:37 -05002776 break;
Peter Zijlstraf1044792012-02-11 06:05:00 +01002777
2778 cfs_rq->next = NULL;
Rik van Riel2c13c9192011-02-01 09:48:37 -05002779 }
Peter Zijlstra2002c692008-11-11 11:52:33 +01002780}
2781
Rik van Rielac53db52011-02-01 09:51:03 -05002782static void __clear_buddies_skip(struct sched_entity *se)
2783{
2784 for_each_sched_entity(se) {
2785 struct cfs_rq *cfs_rq = cfs_rq_of(se);
Peter Zijlstraf1044792012-02-11 06:05:00 +01002786 if (cfs_rq->skip != se)
Rik van Rielac53db52011-02-01 09:51:03 -05002787 break;
Peter Zijlstraf1044792012-02-11 06:05:00 +01002788
2789 cfs_rq->skip = NULL;
Rik van Rielac53db52011-02-01 09:51:03 -05002790 }
2791}
2792
Peter Zijlstraa571bbe2009-01-28 14:51:40 +01002793static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
2794{
Rik van Riel2c13c9192011-02-01 09:48:37 -05002795 if (cfs_rq->last == se)
2796 __clear_buddies_last(se);
2797
2798 if (cfs_rq->next == se)
2799 __clear_buddies_next(se);
Rik van Rielac53db52011-02-01 09:51:03 -05002800
2801 if (cfs_rq->skip == se)
2802 __clear_buddies_skip(se);
Peter Zijlstraa571bbe2009-01-28 14:51:40 +01002803}
2804
Peter Zijlstra6c16a6d2012-03-21 13:07:16 -07002805static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
Paul Turnerd8b49862011-07-21 09:43:41 -07002806
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002807static void
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002808dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002809{
Dmitry Adamushkoa2a2d682007-10-15 17:00:13 +02002810 /*
2811 * Update run-time statistics of the 'current'.
2812 */
2813 update_curr(cfs_rq);
Linus Torvalds17bc14b2012-12-14 07:20:43 -08002814 dequeue_entity_load_avg(cfs_rq, se, flags & DEQUEUE_SLEEP);
Dmitry Adamushkoa2a2d682007-10-15 17:00:13 +02002815
Ingo Molnar19b6a2e2007-08-09 11:16:48 +02002816 update_stats_dequeue(cfs_rq, se);
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002817 if (flags & DEQUEUE_SLEEP) {
Peter Zijlstra67e9fb22007-10-15 17:00:10 +02002818#ifdef CONFIG_SCHEDSTATS
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002819 if (entity_is_task(se)) {
2820 struct task_struct *tsk = task_of(se);
2821
2822 if (tsk->state & TASK_INTERRUPTIBLE)
Frederic Weisbecker78becc22013-04-12 01:51:02 +02002823 se->statistics.sleep_start = rq_clock(rq_of(cfs_rq));
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002824 if (tsk->state & TASK_UNINTERRUPTIBLE)
Frederic Weisbecker78becc22013-04-12 01:51:02 +02002825 se->statistics.block_start = rq_clock(rq_of(cfs_rq));
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002826 }
Dmitry Adamushkodb36cc72007-10-15 17:00:06 +02002827#endif
Peter Zijlstra67e9fb22007-10-15 17:00:10 +02002828 }
2829
Peter Zijlstra2002c692008-11-11 11:52:33 +01002830 clear_buddies(cfs_rq, se);
Peter Zijlstra47932412008-11-04 21:25:09 +01002831
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02002832 if (se != cfs_rq->curr)
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002833 __dequeue_entity(cfs_rq, se);
Linus Torvalds17bc14b2012-12-14 07:20:43 -08002834 se->on_rq = 0;
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002835 account_entity_dequeue(cfs_rq, se);
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01002836
2837 /*
2838 * Normalize the entity after updating the min_vruntime because the
2839 * update can refer to the ->curr item and we need to reflect this
2840 * movement in our normalized position.
2841 */
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002842 if (!(flags & DEQUEUE_SLEEP))
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01002843 se->vruntime -= cfs_rq->min_vruntime;
Peter Zijlstra1e876232011-05-17 16:21:10 -07002844
Paul Turnerd8b49862011-07-21 09:43:41 -07002845 /* return excess runtime on last dequeue */
2846 return_cfs_rq_runtime(cfs_rq);
2847
Peter Zijlstra1e876232011-05-17 16:21:10 -07002848 update_min_vruntime(cfs_rq);
Linus Torvalds17bc14b2012-12-14 07:20:43 -08002849 update_cfs_shares(cfs_rq);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002850}
2851
2852/*
2853 * Preempt the current task with a newly woken task if needed:
2854 */
Peter Zijlstra7c92e542007-09-05 14:32:49 +02002855static void
Ingo Molnar2e09bf52007-10-15 17:00:05 +02002856check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002857{
Peter Zijlstra11697832007-09-05 14:32:49 +02002858 unsigned long ideal_runtime, delta_exec;
Wang Xingchaof4cfb332011-09-16 13:35:52 -04002859 struct sched_entity *se;
2860 s64 delta;
Peter Zijlstra11697832007-09-05 14:32:49 +02002861
Peter Zijlstra6d0f0eb2007-10-15 17:00:05 +02002862 ideal_runtime = sched_slice(cfs_rq, curr);
Peter Zijlstra11697832007-09-05 14:32:49 +02002863 delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
Mike Galbraitha9f3e2b2009-01-28 14:51:39 +01002864 if (delta_exec > ideal_runtime) {
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002865 resched_task(rq_of(cfs_rq)->curr);
Mike Galbraitha9f3e2b2009-01-28 14:51:39 +01002866 /*
2867 * The current task ran long enough, ensure it doesn't get
2868 * re-elected due to buddy favours.
2869 */
2870 clear_buddies(cfs_rq, curr);
Mike Galbraithf685cea2009-10-23 23:09:22 +02002871 return;
2872 }
2873
2874 /*
2875 * Ensure that a task that missed wakeup preemption by a
2876 * narrow margin doesn't have to wait for a full slice.
2877 * This also mitigates buddy induced latencies under load.
2878 */
Mike Galbraithf685cea2009-10-23 23:09:22 +02002879 if (delta_exec < sysctl_sched_min_granularity)
2880 return;
2881
Wang Xingchaof4cfb332011-09-16 13:35:52 -04002882 se = __pick_first_entity(cfs_rq);
2883 delta = curr->vruntime - se->vruntime;
Mike Galbraithf685cea2009-10-23 23:09:22 +02002884
Wang Xingchaof4cfb332011-09-16 13:35:52 -04002885 if (delta < 0)
2886 return;
Mike Galbraithd7d82942011-01-05 05:41:17 +01002887
Wang Xingchaof4cfb332011-09-16 13:35:52 -04002888 if (delta > ideal_runtime)
2889 resched_task(rq_of(cfs_rq)->curr);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002890}
2891
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02002892static void
Ingo Molnar8494f412007-08-09 11:16:48 +02002893set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002894{
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02002895 /* 'current' is not kept within the tree. */
2896 if (se->on_rq) {
2897 /*
2898 * Any task has to be enqueued before it get to execute on
2899 * a CPU. So account for the time it spent waiting on the
2900 * runqueue.
2901 */
2902 update_stats_wait_end(cfs_rq, se);
2903 __dequeue_entity(cfs_rq, se);
2904 }
2905
Ingo Molnar79303e92007-08-09 11:16:47 +02002906 update_stats_curr_start(cfs_rq, se);
Ingo Molnar429d43b2007-10-15 17:00:03 +02002907 cfs_rq->curr = se;
Ingo Molnareba1ed42007-10-15 17:00:02 +02002908#ifdef CONFIG_SCHEDSTATS
2909 /*
2910 * Track our maximum slice length, if the CPU's load is at
2911 * least twice that of our own weight (i.e. dont track it
2912 * when there are only lesser-weight tasks around):
2913 */
Dmitry Adamushko495eca42007-10-15 17:00:06 +02002914 if (rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
Lucas De Marchi41acab82010-03-10 23:37:45 -03002915 se->statistics.slice_max = max(se->statistics.slice_max,
Ingo Molnareba1ed42007-10-15 17:00:02 +02002916 se->sum_exec_runtime - se->prev_sum_exec_runtime);
2917 }
2918#endif
Peter Zijlstra4a55b452007-09-05 14:32:49 +02002919 se->prev_sum_exec_runtime = se->sum_exec_runtime;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002920}
2921
Peter Zijlstra3f3a4902008-10-24 11:06:16 +02002922static int
2923wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
2924
Rik van Rielac53db52011-02-01 09:51:03 -05002925/*
2926 * Pick the next process, keeping these things in mind, in this order:
2927 * 1) keep things fair between processes/task groups
2928 * 2) pick the "next" process, since someone really wants that to run
2929 * 3) pick the "last" process, for cache locality
2930 * 4) do not run the "skip" process, if something else is available
2931 */
Peter Zijlstra678d5712012-02-11 06:05:00 +01002932static struct sched_entity *
2933pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
Peter Zijlstraaa2ac252008-03-14 21:12:12 +01002934{
Peter Zijlstra678d5712012-02-11 06:05:00 +01002935 struct sched_entity *left = __pick_first_entity(cfs_rq);
2936 struct sched_entity *se;
2937
2938 /*
2939 * If curr is set we have to see if its left of the leftmost entity
2940 * still in the tree, provided there was anything in the tree at all.
2941 */
2942 if (!left || (curr && entity_before(curr, left)))
2943 left = curr;
2944
2945 se = left; /* ideally we run the leftmost entity */
Peter Zijlstraf4b67552008-11-04 21:25:07 +01002946
Rik van Rielac53db52011-02-01 09:51:03 -05002947 /*
2948 * Avoid running the skip buddy, if running something else can
2949 * be done without getting too unfair.
2950 */
2951 if (cfs_rq->skip == se) {
Peter Zijlstra678d5712012-02-11 06:05:00 +01002952 struct sched_entity *second;
2953
2954 if (se == curr) {
2955 second = __pick_first_entity(cfs_rq);
2956 } else {
2957 second = __pick_next_entity(se);
2958 if (!second || (curr && entity_before(curr, second)))
2959 second = curr;
2960 }
2961
Rik van Rielac53db52011-02-01 09:51:03 -05002962 if (second && wakeup_preempt_entity(second, left) < 1)
2963 se = second;
2964 }
Peter Zijlstraaa2ac252008-03-14 21:12:12 +01002965
Mike Galbraithf685cea2009-10-23 23:09:22 +02002966 /*
2967 * Prefer last buddy, try to return the CPU to a preempted task.
2968 */
2969 if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
2970 se = cfs_rq->last;
2971
Rik van Rielac53db52011-02-01 09:51:03 -05002972 /*
2973 * Someone really wants this to run. If it's not unfair, run it.
2974 */
2975 if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
2976 se = cfs_rq->next;
2977
Mike Galbraithf685cea2009-10-23 23:09:22 +02002978 clear_buddies(cfs_rq, se);
Peter Zijlstra47932412008-11-04 21:25:09 +01002979
2980 return se;
Peter Zijlstraaa2ac252008-03-14 21:12:12 +01002981}
2982
Peter Zijlstra678d5712012-02-11 06:05:00 +01002983static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
Paul Turnerd3d9dc32011-07-21 09:43:39 -07002984
Ingo Molnarab6cde22007-08-09 11:16:48 +02002985static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002986{
2987 /*
2988 * If still on the runqueue then deactivate_task()
2989 * was not called and update_curr() has to be done:
2990 */
2991 if (prev->on_rq)
Ingo Molnarb7cc0892007-08-09 11:16:47 +02002992 update_curr(cfs_rq);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02002993
Paul Turnerd3d9dc32011-07-21 09:43:39 -07002994 /* throttle cfs_rqs exceeding runtime */
2995 check_cfs_rq_runtime(cfs_rq);
2996
Peter Zijlstraddc97292007-10-15 17:00:10 +02002997 check_spread(cfs_rq, prev);
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02002998 if (prev->on_rq) {
Ingo Molnar5870db52007-08-09 11:16:47 +02002999 update_stats_wait_start(cfs_rq, prev);
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02003000 /* Put 'current' back into the tree. */
3001 __enqueue_entity(cfs_rq, prev);
Paul Turner9d85f212012-10-04 13:18:29 +02003002 /* in !on_rq case, update occurred at dequeue */
Paul Turner9ee474f2012-10-04 13:18:30 +02003003 update_entity_load_avg(prev, 1);
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02003004 }
Ingo Molnar429d43b2007-10-15 17:00:03 +02003005 cfs_rq->curr = NULL;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003006}
3007
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003008static void
3009entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003010{
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003011 /*
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02003012 * Update run-time statistics of the 'current'.
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003013 */
Dmitry Adamushko30cfdcf2007-10-15 17:00:07 +02003014 update_curr(cfs_rq);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003015
Paul Turner43365bd2010-12-15 19:10:17 -08003016 /*
Paul Turner9d85f212012-10-04 13:18:29 +02003017 * Ensure that runnable average is periodically updated.
3018 */
Paul Turner9ee474f2012-10-04 13:18:30 +02003019 update_entity_load_avg(curr, 1);
Paul Turneraff3e492012-10-04 13:18:30 +02003020 update_cfs_rq_blocked_load(cfs_rq, 1);
Peter Zijlstrabf0bd942013-07-26 23:48:42 +02003021 update_cfs_shares(cfs_rq);
Paul Turner9d85f212012-10-04 13:18:29 +02003022
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003023#ifdef CONFIG_SCHED_HRTICK
3024 /*
3025 * queued ticks are scheduled to match the slice, so don't bother
3026 * validating it and just reschedule.
3027 */
Harvey Harrison983ed7a2008-04-24 18:17:55 -07003028 if (queued) {
3029 resched_task(rq_of(cfs_rq)->curr);
3030 return;
3031 }
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003032 /*
3033 * don't let the period tick interfere with the hrtick preemption
3034 */
3035 if (!sched_feat(DOUBLE_TICK) &&
3036 hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
3037 return;
3038#endif
3039
Yong Zhang2c2efae2011-07-29 16:20:33 +08003040 if (cfs_rq->nr_running > 1)
Ingo Molnar2e09bf52007-10-15 17:00:05 +02003041 check_preempt_tick(cfs_rq, curr);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003042}
3043
Paul Turnerab84d312011-07-21 09:43:28 -07003044
3045/**************************************************
3046 * CFS bandwidth control machinery
3047 */
3048
3049#ifdef CONFIG_CFS_BANDWIDTH
Peter Zijlstra029632f2011-10-25 10:00:11 +02003050
3051#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01003052static struct static_key __cfs_bandwidth_used;
Peter Zijlstra029632f2011-10-25 10:00:11 +02003053
3054static inline bool cfs_bandwidth_used(void)
3055{
Ingo Molnarc5905af2012-02-24 08:31:31 +01003056 return static_key_false(&__cfs_bandwidth_used);
Peter Zijlstra029632f2011-10-25 10:00:11 +02003057}
3058
Ben Segall1ee14e62013-10-16 11:16:12 -07003059void cfs_bandwidth_usage_inc(void)
Peter Zijlstra029632f2011-10-25 10:00:11 +02003060{
Ben Segall1ee14e62013-10-16 11:16:12 -07003061 static_key_slow_inc(&__cfs_bandwidth_used);
3062}
3063
3064void cfs_bandwidth_usage_dec(void)
3065{
3066 static_key_slow_dec(&__cfs_bandwidth_used);
Peter Zijlstra029632f2011-10-25 10:00:11 +02003067}
3068#else /* HAVE_JUMP_LABEL */
3069static bool cfs_bandwidth_used(void)
3070{
3071 return true;
3072}
3073
Ben Segall1ee14e62013-10-16 11:16:12 -07003074void cfs_bandwidth_usage_inc(void) {}
3075void cfs_bandwidth_usage_dec(void) {}
Peter Zijlstra029632f2011-10-25 10:00:11 +02003076#endif /* HAVE_JUMP_LABEL */
3077
Paul Turnerab84d312011-07-21 09:43:28 -07003078/*
3079 * default period for cfs group bandwidth.
3080 * default: 0.1s, units: nanoseconds
3081 */
3082static inline u64 default_cfs_period(void)
3083{
3084 return 100000000ULL;
3085}
Paul Turnerec12cb72011-07-21 09:43:30 -07003086
3087static inline u64 sched_cfs_bandwidth_slice(void)
3088{
3089 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
3090}
3091
Paul Turnera9cf55b2011-07-21 09:43:32 -07003092/*
3093 * Replenish runtime according to assigned quota and update expiration time.
3094 * We use sched_clock_cpu directly instead of rq->clock to avoid adding
3095 * additional synchronization around rq->lock.
3096 *
3097 * requires cfs_b->lock
3098 */
Peter Zijlstra029632f2011-10-25 10:00:11 +02003099void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
Paul Turnera9cf55b2011-07-21 09:43:32 -07003100{
3101 u64 now;
3102
3103 if (cfs_b->quota == RUNTIME_INF)
3104 return;
3105
3106 now = sched_clock_cpu(smp_processor_id());
3107 cfs_b->runtime = cfs_b->quota;
3108 cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
3109}
3110
Peter Zijlstra029632f2011-10-25 10:00:11 +02003111static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
3112{
3113 return &tg->cfs_bandwidth;
3114}
3115
Paul Turnerf1b17282012-10-04 13:18:31 +02003116/* rq->task_clock normalized against any time this cfs_rq has spent throttled */
3117static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
3118{
3119 if (unlikely(cfs_rq->throttle_count))
3120 return cfs_rq->throttled_clock_task;
3121
Frederic Weisbecker78becc22013-04-12 01:51:02 +02003122 return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
Paul Turnerf1b17282012-10-04 13:18:31 +02003123}
3124
Paul Turner85dac902011-07-21 09:43:33 -07003125/* returns 0 on failure to allocate runtime */
3126static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
Paul Turnerec12cb72011-07-21 09:43:30 -07003127{
3128 struct task_group *tg = cfs_rq->tg;
3129 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
Paul Turnera9cf55b2011-07-21 09:43:32 -07003130 u64 amount = 0, min_amount, expires;
Paul Turnerec12cb72011-07-21 09:43:30 -07003131
3132 /* note: this is a positive sum as runtime_remaining <= 0 */
3133 min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
3134
3135 raw_spin_lock(&cfs_b->lock);
3136 if (cfs_b->quota == RUNTIME_INF)
3137 amount = min_amount;
Paul Turner58088ad2011-07-21 09:43:31 -07003138 else {
Paul Turnera9cf55b2011-07-21 09:43:32 -07003139 /*
3140 * If the bandwidth pool has become inactive, then at least one
3141 * period must have elapsed since the last consumption.
3142 * Refresh the global state and ensure bandwidth timer becomes
3143 * active.
3144 */
3145 if (!cfs_b->timer_active) {
3146 __refill_cfs_bandwidth_runtime(cfs_b);
Paul Turner58088ad2011-07-21 09:43:31 -07003147 __start_cfs_bandwidth(cfs_b);
Paul Turnera9cf55b2011-07-21 09:43:32 -07003148 }
Paul Turner58088ad2011-07-21 09:43:31 -07003149
3150 if (cfs_b->runtime > 0) {
3151 amount = min(cfs_b->runtime, min_amount);
3152 cfs_b->runtime -= amount;
3153 cfs_b->idle = 0;
3154 }
Paul Turnerec12cb72011-07-21 09:43:30 -07003155 }
Paul Turnera9cf55b2011-07-21 09:43:32 -07003156 expires = cfs_b->runtime_expires;
Paul Turnerec12cb72011-07-21 09:43:30 -07003157 raw_spin_unlock(&cfs_b->lock);
3158
3159 cfs_rq->runtime_remaining += amount;
Paul Turnera9cf55b2011-07-21 09:43:32 -07003160 /*
3161 * we may have advanced our local expiration to account for allowed
3162 * spread between our sched_clock and the one on which runtime was
3163 * issued.
3164 */
3165 if ((s64)(expires - cfs_rq->runtime_expires) > 0)
3166 cfs_rq->runtime_expires = expires;
Paul Turner85dac902011-07-21 09:43:33 -07003167
3168 return cfs_rq->runtime_remaining > 0;
Paul Turnera9cf55b2011-07-21 09:43:32 -07003169}
3170
3171/*
3172 * Note: This depends on the synchronization provided by sched_clock and the
3173 * fact that rq->clock snapshots this value.
3174 */
3175static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq)
3176{
3177 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
Paul Turnera9cf55b2011-07-21 09:43:32 -07003178
3179 /* if the deadline is ahead of our clock, nothing to do */
Frederic Weisbecker78becc22013-04-12 01:51:02 +02003180 if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0))
Paul Turnera9cf55b2011-07-21 09:43:32 -07003181 return;
3182
3183 if (cfs_rq->runtime_remaining < 0)
3184 return;
3185
3186 /*
3187 * If the local deadline has passed we have to consider the
3188 * possibility that our sched_clock is 'fast' and the global deadline
3189 * has not truly expired.
3190 *
3191 * Fortunately we can check determine whether this the case by checking
3192 * whether the global deadline has advanced.
3193 */
3194
3195 if ((s64)(cfs_rq->runtime_expires - cfs_b->runtime_expires) >= 0) {
3196 /* extend local deadline, drift is bounded above by 2 ticks */
3197 cfs_rq->runtime_expires += TICK_NSEC;
3198 } else {
3199 /* global deadline is ahead, expiration has passed */
3200 cfs_rq->runtime_remaining = 0;
3201 }
Paul Turnerec12cb72011-07-21 09:43:30 -07003202}
3203
Peter Zijlstra9dbdb152013-11-18 18:27:06 +01003204static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
Paul Turnerec12cb72011-07-21 09:43:30 -07003205{
Paul Turnera9cf55b2011-07-21 09:43:32 -07003206 /* dock delta_exec before expiring quota (as it could span periods) */
Paul Turnerec12cb72011-07-21 09:43:30 -07003207 cfs_rq->runtime_remaining -= delta_exec;
Paul Turnera9cf55b2011-07-21 09:43:32 -07003208 expire_cfs_rq_runtime(cfs_rq);
3209
3210 if (likely(cfs_rq->runtime_remaining > 0))
Paul Turnerec12cb72011-07-21 09:43:30 -07003211 return;
3212
Paul Turner85dac902011-07-21 09:43:33 -07003213 /*
3214 * if we're unable to extend our runtime we resched so that the active
3215 * hierarchy can be throttled
3216 */
3217 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
3218 resched_task(rq_of(cfs_rq)->curr);
Paul Turnerec12cb72011-07-21 09:43:30 -07003219}
3220
Peter Zijlstra6c16a6d2012-03-21 13:07:16 -07003221static __always_inline
Peter Zijlstra9dbdb152013-11-18 18:27:06 +01003222void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
Paul Turnerec12cb72011-07-21 09:43:30 -07003223{
Paul Turner56f570e2011-11-07 20:26:33 -08003224 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
Paul Turnerec12cb72011-07-21 09:43:30 -07003225 return;
3226
3227 __account_cfs_rq_runtime(cfs_rq, delta_exec);
3228}
3229
Paul Turner85dac902011-07-21 09:43:33 -07003230static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
3231{
Paul Turner56f570e2011-11-07 20:26:33 -08003232 return cfs_bandwidth_used() && cfs_rq->throttled;
Paul Turner85dac902011-07-21 09:43:33 -07003233}
3234
Paul Turner64660c82011-07-21 09:43:36 -07003235/* check whether cfs_rq, or any parent, is throttled */
3236static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
3237{
Paul Turner56f570e2011-11-07 20:26:33 -08003238 return cfs_bandwidth_used() && cfs_rq->throttle_count;
Paul Turner64660c82011-07-21 09:43:36 -07003239}
3240
3241/*
3242 * Ensure that neither of the group entities corresponding to src_cpu or
3243 * dest_cpu are members of a throttled hierarchy when performing group
3244 * load-balance operations.
3245 */
3246static inline int throttled_lb_pair(struct task_group *tg,
3247 int src_cpu, int dest_cpu)
3248{
3249 struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
3250
3251 src_cfs_rq = tg->cfs_rq[src_cpu];
3252 dest_cfs_rq = tg->cfs_rq[dest_cpu];
3253
3254 return throttled_hierarchy(src_cfs_rq) ||
3255 throttled_hierarchy(dest_cfs_rq);
3256}
3257
3258/* updated child weight may affect parent so we have to do this bottom up */
3259static int tg_unthrottle_up(struct task_group *tg, void *data)
3260{
3261 struct rq *rq = data;
3262 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
3263
3264 cfs_rq->throttle_count--;
3265#ifdef CONFIG_SMP
3266 if (!cfs_rq->throttle_count) {
Paul Turnerf1b17282012-10-04 13:18:31 +02003267 /* adjust cfs_rq_clock_task() */
Frederic Weisbecker78becc22013-04-12 01:51:02 +02003268 cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
Paul Turnerf1b17282012-10-04 13:18:31 +02003269 cfs_rq->throttled_clock_task;
Paul Turner64660c82011-07-21 09:43:36 -07003270 }
3271#endif
3272
3273 return 0;
3274}
3275
3276static int tg_throttle_down(struct task_group *tg, void *data)
3277{
3278 struct rq *rq = data;
3279 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
3280
Paul Turner82958362012-10-04 13:18:31 +02003281 /* group is entering throttled state, stop time */
3282 if (!cfs_rq->throttle_count)
Frederic Weisbecker78becc22013-04-12 01:51:02 +02003283 cfs_rq->throttled_clock_task = rq_clock_task(rq);
Paul Turner64660c82011-07-21 09:43:36 -07003284 cfs_rq->throttle_count++;
3285
3286 return 0;
3287}
3288
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003289static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
Paul Turner85dac902011-07-21 09:43:33 -07003290{
3291 struct rq *rq = rq_of(cfs_rq);
3292 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
3293 struct sched_entity *se;
3294 long task_delta, dequeue = 1;
3295
3296 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
3297
Paul Turnerf1b17282012-10-04 13:18:31 +02003298 /* freeze hierarchy runnable averages while throttled */
Paul Turner64660c82011-07-21 09:43:36 -07003299 rcu_read_lock();
3300 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
3301 rcu_read_unlock();
Paul Turner85dac902011-07-21 09:43:33 -07003302
3303 task_delta = cfs_rq->h_nr_running;
3304 for_each_sched_entity(se) {
3305 struct cfs_rq *qcfs_rq = cfs_rq_of(se);
3306 /* throttled entity or throttle-on-deactivate */
3307 if (!se->on_rq)
3308 break;
3309
3310 if (dequeue)
3311 dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
3312 qcfs_rq->h_nr_running -= task_delta;
3313
3314 if (qcfs_rq->load.weight)
3315 dequeue = 0;
3316 }
3317
3318 if (!se)
3319 rq->nr_running -= task_delta;
3320
3321 cfs_rq->throttled = 1;
Frederic Weisbecker78becc22013-04-12 01:51:02 +02003322 cfs_rq->throttled_clock = rq_clock(rq);
Paul Turner85dac902011-07-21 09:43:33 -07003323 raw_spin_lock(&cfs_b->lock);
3324 list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
Ben Segallf9f9ffc2013-10-16 11:16:32 -07003325 if (!cfs_b->timer_active)
3326 __start_cfs_bandwidth(cfs_b);
Paul Turner85dac902011-07-21 09:43:33 -07003327 raw_spin_unlock(&cfs_b->lock);
3328}
3329
Peter Zijlstra029632f2011-10-25 10:00:11 +02003330void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
Paul Turner671fd9d2011-07-21 09:43:34 -07003331{
3332 struct rq *rq = rq_of(cfs_rq);
3333 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
3334 struct sched_entity *se;
3335 int enqueue = 1;
3336 long task_delta;
3337
Michael Wang22b958d2013-06-04 14:23:39 +08003338 se = cfs_rq->tg->se[cpu_of(rq)];
Paul Turner671fd9d2011-07-21 09:43:34 -07003339
3340 cfs_rq->throttled = 0;
Frederic Weisbecker1a55af22013-04-12 01:51:01 +02003341
3342 update_rq_clock(rq);
3343
Paul Turner671fd9d2011-07-21 09:43:34 -07003344 raw_spin_lock(&cfs_b->lock);
Frederic Weisbecker78becc22013-04-12 01:51:02 +02003345 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
Paul Turner671fd9d2011-07-21 09:43:34 -07003346 list_del_rcu(&cfs_rq->throttled_list);
3347 raw_spin_unlock(&cfs_b->lock);
3348
Paul Turner64660c82011-07-21 09:43:36 -07003349 /* update hierarchical throttle state */
3350 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
3351
Paul Turner671fd9d2011-07-21 09:43:34 -07003352 if (!cfs_rq->load.weight)
3353 return;
3354
3355 task_delta = cfs_rq->h_nr_running;
3356 for_each_sched_entity(se) {
3357 if (se->on_rq)
3358 enqueue = 0;
3359
3360 cfs_rq = cfs_rq_of(se);
3361 if (enqueue)
3362 enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
3363 cfs_rq->h_nr_running += task_delta;
3364
3365 if (cfs_rq_throttled(cfs_rq))
3366 break;
3367 }
3368
3369 if (!se)
3370 rq->nr_running += task_delta;
3371
3372 /* determine whether we need to wake up potentially idle cpu */
3373 if (rq->curr == rq->idle && rq->cfs.nr_running)
3374 resched_task(rq->curr);
3375}
3376
3377static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
3378 u64 remaining, u64 expires)
3379{
3380 struct cfs_rq *cfs_rq;
3381 u64 runtime = remaining;
3382
3383 rcu_read_lock();
3384 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
3385 throttled_list) {
3386 struct rq *rq = rq_of(cfs_rq);
3387
3388 raw_spin_lock(&rq->lock);
3389 if (!cfs_rq_throttled(cfs_rq))
3390 goto next;
3391
3392 runtime = -cfs_rq->runtime_remaining + 1;
3393 if (runtime > remaining)
3394 runtime = remaining;
3395 remaining -= runtime;
3396
3397 cfs_rq->runtime_remaining += runtime;
3398 cfs_rq->runtime_expires = expires;
3399
3400 /* we check whether we're throttled above */
3401 if (cfs_rq->runtime_remaining > 0)
3402 unthrottle_cfs_rq(cfs_rq);
3403
3404next:
3405 raw_spin_unlock(&rq->lock);
3406
3407 if (!remaining)
3408 break;
3409 }
3410 rcu_read_unlock();
3411
3412 return remaining;
3413}
3414
Paul Turner58088ad2011-07-21 09:43:31 -07003415/*
3416 * Responsible for refilling a task_group's bandwidth and unthrottling its
3417 * cfs_rqs as appropriate. If there has been no activity within the last
3418 * period the timer is deactivated until scheduling resumes; cfs_b->idle is
3419 * used to track this state.
3420 */
3421static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
3422{
Paul Turner671fd9d2011-07-21 09:43:34 -07003423 u64 runtime, runtime_expires;
3424 int idle = 1, throttled;
Paul Turner58088ad2011-07-21 09:43:31 -07003425
3426 raw_spin_lock(&cfs_b->lock);
3427 /* no need to continue the timer with no bandwidth constraint */
3428 if (cfs_b->quota == RUNTIME_INF)
3429 goto out_unlock;
3430
Paul Turner671fd9d2011-07-21 09:43:34 -07003431 throttled = !list_empty(&cfs_b->throttled_cfs_rq);
3432 /* idle depends on !throttled (for the case of a large deficit) */
3433 idle = cfs_b->idle && !throttled;
Nikhil Raoe8da1b12011-07-21 09:43:40 -07003434 cfs_b->nr_periods += overrun;
Paul Turner671fd9d2011-07-21 09:43:34 -07003435
Paul Turnera9cf55b2011-07-21 09:43:32 -07003436 /* if we're going inactive then everything else can be deferred */
3437 if (idle)
3438 goto out_unlock;
3439
Ben Segall927b54f2013-10-16 11:16:22 -07003440 /*
3441 * if we have relooped after returning idle once, we need to update our
3442 * status as actually running, so that other cpus doing
3443 * __start_cfs_bandwidth will stop trying to cancel us.
3444 */
3445 cfs_b->timer_active = 1;
3446
Paul Turnera9cf55b2011-07-21 09:43:32 -07003447 __refill_cfs_bandwidth_runtime(cfs_b);
3448
Paul Turner671fd9d2011-07-21 09:43:34 -07003449 if (!throttled) {
3450 /* mark as potentially idle for the upcoming period */
3451 cfs_b->idle = 1;
3452 goto out_unlock;
3453 }
Paul Turner58088ad2011-07-21 09:43:31 -07003454
Nikhil Raoe8da1b12011-07-21 09:43:40 -07003455 /* account preceding periods in which throttling occurred */
3456 cfs_b->nr_throttled += overrun;
3457
Paul Turner671fd9d2011-07-21 09:43:34 -07003458 /*
3459 * There are throttled entities so we must first use the new bandwidth
3460 * to unthrottle them before making it generally available. This
3461 * ensures that all existing debts will be paid before a new cfs_rq is
3462 * allowed to run.
3463 */
3464 runtime = cfs_b->runtime;
3465 runtime_expires = cfs_b->runtime_expires;
3466 cfs_b->runtime = 0;
3467
3468 /*
3469 * This check is repeated as we are holding onto the new bandwidth
3470 * while we unthrottle. This can potentially race with an unthrottled
3471 * group trying to acquire new bandwidth from the global pool.
3472 */
3473 while (throttled && runtime > 0) {
3474 raw_spin_unlock(&cfs_b->lock);
3475 /* we can't nest cfs_b->lock while distributing bandwidth */
3476 runtime = distribute_cfs_runtime(cfs_b, runtime,
3477 runtime_expires);
3478 raw_spin_lock(&cfs_b->lock);
3479
3480 throttled = !list_empty(&cfs_b->throttled_cfs_rq);
3481 }
3482
3483 /* return (any) remaining runtime */
3484 cfs_b->runtime = runtime;
3485 /*
3486 * While we are ensured activity in the period following an
3487 * unthrottle, this also covers the case in which the new bandwidth is
3488 * insufficient to cover the existing bandwidth deficit. (Forcing the
3489 * timer to remain active while there are any throttled entities.)
3490 */
3491 cfs_b->idle = 0;
Paul Turner58088ad2011-07-21 09:43:31 -07003492out_unlock:
3493 if (idle)
3494 cfs_b->timer_active = 0;
3495 raw_spin_unlock(&cfs_b->lock);
3496
3497 return idle;
3498}
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003499
Paul Turnerd8b49862011-07-21 09:43:41 -07003500/* a cfs_rq won't donate quota below this amount */
3501static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
3502/* minimum remaining period time to redistribute slack quota */
3503static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
3504/* how long we wait to gather additional slack before distributing */
3505static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
3506
Ben Segalldb06e782013-10-16 11:16:17 -07003507/*
3508 * Are we near the end of the current quota period?
3509 *
3510 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
3511 * hrtimer base being cleared by __hrtimer_start_range_ns. In the case of
3512 * migrate_hrtimers, base is never cleared, so we are fine.
3513 */
Paul Turnerd8b49862011-07-21 09:43:41 -07003514static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
3515{
3516 struct hrtimer *refresh_timer = &cfs_b->period_timer;
3517 u64 remaining;
3518
3519 /* if the call-back is running a quota refresh is already occurring */
3520 if (hrtimer_callback_running(refresh_timer))
3521 return 1;
3522
3523 /* is a quota refresh about to occur? */
3524 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
3525 if (remaining < min_expire)
3526 return 1;
3527
3528 return 0;
3529}
3530
3531static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
3532{
3533 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
3534
3535 /* if there's a quota refresh soon don't bother with slack */
3536 if (runtime_refresh_within(cfs_b, min_left))
3537 return;
3538
3539 start_bandwidth_timer(&cfs_b->slack_timer,
3540 ns_to_ktime(cfs_bandwidth_slack_period));
3541}
3542
3543/* we know any runtime found here is valid as update_curr() precedes return */
3544static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
3545{
3546 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
3547 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
3548
3549 if (slack_runtime <= 0)
3550 return;
3551
3552 raw_spin_lock(&cfs_b->lock);
3553 if (cfs_b->quota != RUNTIME_INF &&
3554 cfs_rq->runtime_expires == cfs_b->runtime_expires) {
3555 cfs_b->runtime += slack_runtime;
3556
3557 /* we are under rq->lock, defer unthrottling using a timer */
3558 if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
3559 !list_empty(&cfs_b->throttled_cfs_rq))
3560 start_cfs_slack_bandwidth(cfs_b);
3561 }
3562 raw_spin_unlock(&cfs_b->lock);
3563
3564 /* even if it's not valid for return we don't want to try again */
3565 cfs_rq->runtime_remaining -= slack_runtime;
3566}
3567
3568static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
3569{
Paul Turner56f570e2011-11-07 20:26:33 -08003570 if (!cfs_bandwidth_used())
3571 return;
3572
Paul Turnerfccfdc62011-11-07 20:26:34 -08003573 if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
Paul Turnerd8b49862011-07-21 09:43:41 -07003574 return;
3575
3576 __return_cfs_rq_runtime(cfs_rq);
3577}
3578
3579/*
3580 * This is done with a timer (instead of inline with bandwidth return) since
3581 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
3582 */
3583static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
3584{
3585 u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
3586 u64 expires;
3587
3588 /* confirm we're still not at a refresh boundary */
Paul Turnerd8b49862011-07-21 09:43:41 -07003589 raw_spin_lock(&cfs_b->lock);
Ben Segalldb06e782013-10-16 11:16:17 -07003590 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
3591 raw_spin_unlock(&cfs_b->lock);
3592 return;
3593 }
3594
Paul Turnerd8b49862011-07-21 09:43:41 -07003595 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) {
3596 runtime = cfs_b->runtime;
3597 cfs_b->runtime = 0;
3598 }
3599 expires = cfs_b->runtime_expires;
3600 raw_spin_unlock(&cfs_b->lock);
3601
3602 if (!runtime)
3603 return;
3604
3605 runtime = distribute_cfs_runtime(cfs_b, runtime, expires);
3606
3607 raw_spin_lock(&cfs_b->lock);
3608 if (expires == cfs_b->runtime_expires)
3609 cfs_b->runtime = runtime;
3610 raw_spin_unlock(&cfs_b->lock);
3611}
3612
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003613/*
3614 * When a group wakes up we want to make sure that its quota is not already
3615 * expired/exceeded, otherwise it may be allowed to steal additional ticks of
3616 * runtime as update_curr() throttling can not not trigger until it's on-rq.
3617 */
3618static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
3619{
Paul Turner56f570e2011-11-07 20:26:33 -08003620 if (!cfs_bandwidth_used())
3621 return;
3622
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003623 /* an active group must be handled by the update_curr()->put() path */
3624 if (!cfs_rq->runtime_enabled || cfs_rq->curr)
3625 return;
3626
3627 /* ensure the group is not already throttled */
3628 if (cfs_rq_throttled(cfs_rq))
3629 return;
3630
3631 /* update runtime allocation */
3632 account_cfs_rq_runtime(cfs_rq, 0);
3633 if (cfs_rq->runtime_remaining <= 0)
3634 throttle_cfs_rq(cfs_rq);
3635}
3636
3637/* conditionally throttle active cfs_rq's from put_prev_entity() */
Peter Zijlstra678d5712012-02-11 06:05:00 +01003638static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003639{
Paul Turner56f570e2011-11-07 20:26:33 -08003640 if (!cfs_bandwidth_used())
Peter Zijlstra678d5712012-02-11 06:05:00 +01003641 return false;
Paul Turner56f570e2011-11-07 20:26:33 -08003642
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003643 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
Peter Zijlstra678d5712012-02-11 06:05:00 +01003644 return false;
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003645
3646 /*
3647 * it's possible for a throttled entity to be forced into a running
3648 * state (e.g. set_curr_task), in this case we're finished.
3649 */
3650 if (cfs_rq_throttled(cfs_rq))
Peter Zijlstra678d5712012-02-11 06:05:00 +01003651 return true;
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003652
3653 throttle_cfs_rq(cfs_rq);
Peter Zijlstra678d5712012-02-11 06:05:00 +01003654 return true;
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003655}
Peter Zijlstra029632f2011-10-25 10:00:11 +02003656
Peter Zijlstra029632f2011-10-25 10:00:11 +02003657static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
3658{
3659 struct cfs_bandwidth *cfs_b =
3660 container_of(timer, struct cfs_bandwidth, slack_timer);
3661 do_sched_cfs_slack_timer(cfs_b);
3662
3663 return HRTIMER_NORESTART;
3664}
3665
3666static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
3667{
3668 struct cfs_bandwidth *cfs_b =
3669 container_of(timer, struct cfs_bandwidth, period_timer);
3670 ktime_t now;
3671 int overrun;
3672 int idle = 0;
3673
3674 for (;;) {
3675 now = hrtimer_cb_get_time(timer);
3676 overrun = hrtimer_forward(timer, now, cfs_b->period);
3677
3678 if (!overrun)
3679 break;
3680
3681 idle = do_sched_cfs_period_timer(cfs_b, overrun);
3682 }
3683
3684 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
3685}
3686
3687void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
3688{
3689 raw_spin_lock_init(&cfs_b->lock);
3690 cfs_b->runtime = 0;
3691 cfs_b->quota = RUNTIME_INF;
3692 cfs_b->period = ns_to_ktime(default_cfs_period());
3693
3694 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
3695 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3696 cfs_b->period_timer.function = sched_cfs_period_timer;
3697 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3698 cfs_b->slack_timer.function = sched_cfs_slack_timer;
3699}
3700
3701static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
3702{
3703 cfs_rq->runtime_enabled = 0;
3704 INIT_LIST_HEAD(&cfs_rq->throttled_list);
3705}
3706
3707/* requires cfs_b->lock, may release to reprogram timer */
3708void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
3709{
3710 /*
3711 * The timer may be active because we're trying to set a new bandwidth
3712 * period or because we're racing with the tear-down path
3713 * (timer_active==0 becomes visible before the hrtimer call-back
3714 * terminates). In either case we ensure that it's re-programmed
3715 */
Ben Segall927b54f2013-10-16 11:16:22 -07003716 while (unlikely(hrtimer_active(&cfs_b->period_timer)) &&
3717 hrtimer_try_to_cancel(&cfs_b->period_timer) < 0) {
3718 /* bounce the lock to allow do_sched_cfs_period_timer to run */
Peter Zijlstra029632f2011-10-25 10:00:11 +02003719 raw_spin_unlock(&cfs_b->lock);
Ben Segall927b54f2013-10-16 11:16:22 -07003720 cpu_relax();
Peter Zijlstra029632f2011-10-25 10:00:11 +02003721 raw_spin_lock(&cfs_b->lock);
3722 /* if someone else restarted the timer then we're done */
3723 if (cfs_b->timer_active)
3724 return;
3725 }
3726
3727 cfs_b->timer_active = 1;
3728 start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);
3729}
3730
3731static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
3732{
3733 hrtimer_cancel(&cfs_b->period_timer);
3734 hrtimer_cancel(&cfs_b->slack_timer);
3735}
3736
Arnd Bergmann38dc3342013-01-25 14:14:22 +00003737static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
Peter Zijlstra029632f2011-10-25 10:00:11 +02003738{
3739 struct cfs_rq *cfs_rq;
3740
3741 for_each_leaf_cfs_rq(rq, cfs_rq) {
3742 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
3743
3744 if (!cfs_rq->runtime_enabled)
3745 continue;
3746
3747 /*
3748 * clock_task is not advancing so we just need to make sure
3749 * there's some valid quota amount
3750 */
3751 cfs_rq->runtime_remaining = cfs_b->quota;
3752 if (cfs_rq_throttled(cfs_rq))
3753 unthrottle_cfs_rq(cfs_rq);
3754 }
3755}
3756
3757#else /* CONFIG_CFS_BANDWIDTH */
Paul Turnerf1b17282012-10-04 13:18:31 +02003758static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
3759{
Frederic Weisbecker78becc22013-04-12 01:51:02 +02003760 return rq_clock_task(rq_of(cfs_rq));
Paul Turnerf1b17282012-10-04 13:18:31 +02003761}
3762
Peter Zijlstra9dbdb152013-11-18 18:27:06 +01003763static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
Peter Zijlstra678d5712012-02-11 06:05:00 +01003764static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
Paul Turnerd3d9dc32011-07-21 09:43:39 -07003765static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
Peter Zijlstra6c16a6d2012-03-21 13:07:16 -07003766static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
Paul Turner85dac902011-07-21 09:43:33 -07003767
3768static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
3769{
3770 return 0;
3771}
Paul Turner64660c82011-07-21 09:43:36 -07003772
3773static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
3774{
3775 return 0;
3776}
3777
3778static inline int throttled_lb_pair(struct task_group *tg,
3779 int src_cpu, int dest_cpu)
3780{
3781 return 0;
3782}
Peter Zijlstra029632f2011-10-25 10:00:11 +02003783
3784void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
3785
3786#ifdef CONFIG_FAIR_GROUP_SCHED
3787static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
Paul Turnerab84d312011-07-21 09:43:28 -07003788#endif
3789
Peter Zijlstra029632f2011-10-25 10:00:11 +02003790static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
3791{
3792 return NULL;
3793}
3794static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
Peter Boonstoppela4c96ae2012-08-09 15:34:47 -07003795static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
Peter Zijlstra029632f2011-10-25 10:00:11 +02003796
3797#endif /* CONFIG_CFS_BANDWIDTH */
3798
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003799/**************************************************
3800 * CFS operations on tasks:
3801 */
3802
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003803#ifdef CONFIG_SCHED_HRTICK
3804static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
3805{
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003806 struct sched_entity *se = &p->se;
3807 struct cfs_rq *cfs_rq = cfs_rq_of(se);
3808
3809 WARN_ON(task_rq(p) != rq);
3810
Mike Galbraithb39e66e2011-11-22 15:20:07 +01003811 if (cfs_rq->nr_running > 1) {
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003812 u64 slice = sched_slice(cfs_rq, se);
3813 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
3814 s64 delta = slice - ran;
3815
3816 if (delta < 0) {
3817 if (rq->curr == p)
3818 resched_task(p);
3819 return;
3820 }
3821
3822 /*
3823 * Don't schedule slices shorter than 10000ns, that just
3824 * doesn't make sense. Rely on vruntime for fairness.
3825 */
Peter Zijlstra31656512008-07-18 18:01:23 +02003826 if (rq->curr != p)
Peter Zijlstra157124c2008-07-28 11:53:11 +02003827 delta = max_t(s64, 10000LL, delta);
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003828
Peter Zijlstra31656512008-07-18 18:01:23 +02003829 hrtick_start(rq, delta);
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003830 }
3831}
Peter Zijlstraa4c2f002008-10-17 19:27:03 +02003832
3833/*
3834 * called from enqueue/dequeue and updates the hrtick when the
3835 * current task is from our class and nr_running is low enough
3836 * to matter.
3837 */
3838static void hrtick_update(struct rq *rq)
3839{
3840 struct task_struct *curr = rq->curr;
3841
Mike Galbraithb39e66e2011-11-22 15:20:07 +01003842 if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class)
Peter Zijlstraa4c2f002008-10-17 19:27:03 +02003843 return;
3844
3845 if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
3846 hrtick_start_fair(rq, curr);
3847}
Dhaval Giani55e12e52008-06-24 23:39:43 +05303848#else /* !CONFIG_SCHED_HRTICK */
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003849static inline void
3850hrtick_start_fair(struct rq *rq, struct task_struct *p)
3851{
3852}
Peter Zijlstraa4c2f002008-10-17 19:27:03 +02003853
3854static inline void hrtick_update(struct rq *rq)
3855{
3856}
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003857#endif
3858
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003859/*
3860 * The enqueue_task method is called before nr_running is
3861 * increased. Here we update the fair scheduling stats and
3862 * then put the task into the rbtree:
3863 */
Thomas Gleixnerea87bb72010-01-20 20:58:57 +00003864static void
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01003865enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003866{
3867 struct cfs_rq *cfs_rq;
Peter Zijlstra62fb1852008-02-25 17:34:02 +01003868 struct sched_entity *se = &p->se;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003869
3870 for_each_sched_entity(se) {
Peter Zijlstra62fb1852008-02-25 17:34:02 +01003871 if (se->on_rq)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003872 break;
3873 cfs_rq = cfs_rq_of(se);
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01003874 enqueue_entity(cfs_rq, se, flags);
Paul Turner85dac902011-07-21 09:43:33 -07003875
3876 /*
3877 * end evaluation on encountering a throttled cfs_rq
3878 *
3879 * note: in the case of encountering a throttled cfs_rq we will
3880 * post the final h_nr_running increment below.
3881 */
3882 if (cfs_rq_throttled(cfs_rq))
3883 break;
Paul Turner953bfcd2011-07-21 09:43:27 -07003884 cfs_rq->h_nr_running++;
Paul Turner85dac902011-07-21 09:43:33 -07003885
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01003886 flags = ENQUEUE_WAKEUP;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003887 }
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003888
Peter Zijlstra2069dd72010-11-15 15:47:00 -08003889 for_each_sched_entity(se) {
Lin Ming0f317142011-07-22 09:14:31 +08003890 cfs_rq = cfs_rq_of(se);
Paul Turner953bfcd2011-07-21 09:43:27 -07003891 cfs_rq->h_nr_running++;
Peter Zijlstra2069dd72010-11-15 15:47:00 -08003892
Paul Turner85dac902011-07-21 09:43:33 -07003893 if (cfs_rq_throttled(cfs_rq))
3894 break;
3895
Linus Torvalds17bc14b2012-12-14 07:20:43 -08003896 update_cfs_shares(cfs_rq);
Paul Turner9ee474f2012-10-04 13:18:30 +02003897 update_entity_load_avg(se, 1);
Peter Zijlstra2069dd72010-11-15 15:47:00 -08003898 }
3899
Ben Segall18bf2802012-10-04 12:51:20 +02003900 if (!se) {
3901 update_rq_runnable_avg(rq, rq->nr_running);
Paul Turner85dac902011-07-21 09:43:33 -07003902 inc_nr_running(rq);
Ben Segall18bf2802012-10-04 12:51:20 +02003903 }
Peter Zijlstraa4c2f002008-10-17 19:27:03 +02003904 hrtick_update(rq);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003905}
3906
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07003907static void set_next_buddy(struct sched_entity *se);
3908
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003909/*
3910 * The dequeue_task method is called before nr_running is
3911 * decreased. We remove the task from the rbtree and
3912 * update the fair scheduling stats:
3913 */
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01003914static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003915{
3916 struct cfs_rq *cfs_rq;
Peter Zijlstra62fb1852008-02-25 17:34:02 +01003917 struct sched_entity *se = &p->se;
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07003918 int task_sleep = flags & DEQUEUE_SLEEP;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003919
3920 for_each_sched_entity(se) {
3921 cfs_rq = cfs_rq_of(se);
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01003922 dequeue_entity(cfs_rq, se, flags);
Paul Turner85dac902011-07-21 09:43:33 -07003923
3924 /*
3925 * end evaluation on encountering a throttled cfs_rq
3926 *
3927 * note: in the case of encountering a throttled cfs_rq we will
3928 * post the final h_nr_running decrement below.
3929 */
3930 if (cfs_rq_throttled(cfs_rq))
3931 break;
Paul Turner953bfcd2011-07-21 09:43:27 -07003932 cfs_rq->h_nr_running--;
Peter Zijlstra2069dd72010-11-15 15:47:00 -08003933
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003934 /* Don't dequeue parent if it has other entities besides us */
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07003935 if (cfs_rq->load.weight) {
3936 /*
3937 * Bias pick_next to pick a task from this cfs_rq, as
3938 * p is sleeping when it is within its sched_slice.
3939 */
3940 if (task_sleep && parent_entity(se))
3941 set_next_buddy(parent_entity(se));
Paul Turner9598c822011-07-06 22:30:37 -07003942
3943 /* avoid re-evaluating load for this entity */
3944 se = parent_entity(se);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003945 break;
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07003946 }
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01003947 flags |= DEQUEUE_SLEEP;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003948 }
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003949
Peter Zijlstra2069dd72010-11-15 15:47:00 -08003950 for_each_sched_entity(se) {
Lin Ming0f317142011-07-22 09:14:31 +08003951 cfs_rq = cfs_rq_of(se);
Paul Turner953bfcd2011-07-21 09:43:27 -07003952 cfs_rq->h_nr_running--;
Peter Zijlstra2069dd72010-11-15 15:47:00 -08003953
Paul Turner85dac902011-07-21 09:43:33 -07003954 if (cfs_rq_throttled(cfs_rq))
3955 break;
3956
Linus Torvalds17bc14b2012-12-14 07:20:43 -08003957 update_cfs_shares(cfs_rq);
Paul Turner9ee474f2012-10-04 13:18:30 +02003958 update_entity_load_avg(se, 1);
Peter Zijlstra2069dd72010-11-15 15:47:00 -08003959 }
3960
Ben Segall18bf2802012-10-04 12:51:20 +02003961 if (!se) {
Paul Turner85dac902011-07-21 09:43:33 -07003962 dec_nr_running(rq);
Ben Segall18bf2802012-10-04 12:51:20 +02003963 update_rq_runnable_avg(rq, 1);
3964 }
Peter Zijlstraa4c2f002008-10-17 19:27:03 +02003965 hrtick_update(rq);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02003966}
3967
Gregory Haskinse7693a32008-01-25 21:08:09 +01003968#ifdef CONFIG_SMP
Peter Zijlstra029632f2011-10-25 10:00:11 +02003969/* Used instead of source_load when we know the type == 0 */
3970static unsigned long weighted_cpuload(const int cpu)
3971{
Alex Shib92486c2013-06-20 10:18:50 +08003972 return cpu_rq(cpu)->cfs.runnable_load_avg;
Peter Zijlstra029632f2011-10-25 10:00:11 +02003973}
3974
3975/*
3976 * Return a low guess at the load of a migration-source cpu weighted
3977 * according to the scheduling class and "nice" value.
3978 *
3979 * We want to under-estimate the load of migration sources, to
3980 * balance conservatively.
3981 */
3982static unsigned long source_load(int cpu, int type)
3983{
3984 struct rq *rq = cpu_rq(cpu);
3985 unsigned long total = weighted_cpuload(cpu);
3986
3987 if (type == 0 || !sched_feat(LB_BIAS))
3988 return total;
3989
3990 return min(rq->cpu_load[type-1], total);
3991}
3992
3993/*
3994 * Return a high guess at the load of a migration-target cpu weighted
3995 * according to the scheduling class and "nice" value.
3996 */
3997static unsigned long target_load(int cpu, int type)
3998{
3999 struct rq *rq = cpu_rq(cpu);
4000 unsigned long total = weighted_cpuload(cpu);
4001
4002 if (type == 0 || !sched_feat(LB_BIAS))
4003 return total;
4004
4005 return max(rq->cpu_load[type-1], total);
4006}
4007
4008static unsigned long power_of(int cpu)
4009{
4010 return cpu_rq(cpu)->cpu_power;
4011}
4012
4013static unsigned long cpu_avg_load_per_task(int cpu)
4014{
4015 struct rq *rq = cpu_rq(cpu);
4016 unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
Alex Shib92486c2013-06-20 10:18:50 +08004017 unsigned long load_avg = rq->cfs.runnable_load_avg;
Peter Zijlstra029632f2011-10-25 10:00:11 +02004018
4019 if (nr_running)
Alex Shib92486c2013-06-20 10:18:50 +08004020 return load_avg / nr_running;
Peter Zijlstra029632f2011-10-25 10:00:11 +02004021
4022 return 0;
4023}
4024
Michael Wang62470412013-07-04 12:55:51 +08004025static void record_wakee(struct task_struct *p)
4026{
4027 /*
4028 * Rough decay (wiping) for cost saving, don't worry
4029 * about the boundary, really active task won't care
4030 * about the loss.
4031 */
4032 if (jiffies > current->wakee_flip_decay_ts + HZ) {
4033 current->wakee_flips = 0;
4034 current->wakee_flip_decay_ts = jiffies;
4035 }
4036
4037 if (current->last_wakee != p) {
4038 current->last_wakee = p;
4039 current->wakee_flips++;
4040 }
4041}
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004042
Peter Zijlstra74f8e4b2011-04-05 17:23:47 +02004043static void task_waking_fair(struct task_struct *p)
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01004044{
4045 struct sched_entity *se = &p->se;
4046 struct cfs_rq *cfs_rq = cfs_rq_of(se);
Peter Zijlstra3fe16982011-04-05 17:23:48 +02004047 u64 min_vruntime;
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01004048
Peter Zijlstra3fe16982011-04-05 17:23:48 +02004049#ifndef CONFIG_64BIT
4050 u64 min_vruntime_copy;
Peter Zijlstra74f8e4b2011-04-05 17:23:47 +02004051
Peter Zijlstra3fe16982011-04-05 17:23:48 +02004052 do {
4053 min_vruntime_copy = cfs_rq->min_vruntime_copy;
4054 smp_rmb();
4055 min_vruntime = cfs_rq->min_vruntime;
4056 } while (min_vruntime != min_vruntime_copy);
4057#else
4058 min_vruntime = cfs_rq->min_vruntime;
4059#endif
4060
4061 se->vruntime -= min_vruntime;
Michael Wang62470412013-07-04 12:55:51 +08004062 record_wakee(p);
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01004063}
4064
Peter Zijlstrabb3469a2008-06-27 13:41:27 +02004065#ifdef CONFIG_FAIR_GROUP_SCHED
Peter Zijlstraf5bfb7d2008-06-27 13:41:39 +02004066/*
4067 * effective_load() calculates the load change as seen from the root_task_group
4068 *
4069 * Adding load to a group doesn't make a group heavier, but can cause movement
4070 * of group shares between cpus. Assuming the shares were perfectly aligned one
4071 * can calculate the shift in shares.
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004072 *
4073 * Calculate the effective load difference if @wl is added (subtracted) to @tg
4074 * on this @cpu and results in a total addition (subtraction) of @wg to the
4075 * total group weight.
4076 *
4077 * Given a runqueue weight distribution (rw_i) we can compute a shares
4078 * distribution (s_i) using:
4079 *
4080 * s_i = rw_i / \Sum rw_j (1)
4081 *
4082 * Suppose we have 4 CPUs and our @tg is a direct child of the root group and
4083 * has 7 equal weight tasks, distributed as below (rw_i), with the resulting
4084 * shares distribution (s_i):
4085 *
4086 * rw_i = { 2, 4, 1, 0 }
4087 * s_i = { 2/7, 4/7, 1/7, 0 }
4088 *
4089 * As per wake_affine() we're interested in the load of two CPUs (the CPU the
4090 * task used to run on and the CPU the waker is running on), we need to
4091 * compute the effect of waking a task on either CPU and, in case of a sync
4092 * wakeup, compute the effect of the current task going to sleep.
4093 *
4094 * So for a change of @wl to the local @cpu with an overall group weight change
4095 * of @wl we can compute the new shares distribution (s'_i) using:
4096 *
4097 * s'_i = (rw_i + @wl) / (@wg + \Sum rw_j) (2)
4098 *
4099 * Suppose we're interested in CPUs 0 and 1, and want to compute the load
4100 * differences in waking a task to CPU 0. The additional task changes the
4101 * weight and shares distributions like:
4102 *
4103 * rw'_i = { 3, 4, 1, 0 }
4104 * s'_i = { 3/8, 4/8, 1/8, 0 }
4105 *
4106 * We can then compute the difference in effective weight by using:
4107 *
4108 * dw_i = S * (s'_i - s_i) (3)
4109 *
4110 * Where 'S' is the group weight as seen by its parent.
4111 *
4112 * Therefore the effective change in loads on CPU 0 would be 5/56 (3/8 - 2/7)
4113 * times the weight of the group. The effect on CPU 1 would be -4/56 (4/8 -
4114 * 4/7) times the weight of the group.
Peter Zijlstraf5bfb7d2008-06-27 13:41:39 +02004115 */
Peter Zijlstra2069dd72010-11-15 15:47:00 -08004116static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
Peter Zijlstrabb3469a2008-06-27 13:41:27 +02004117{
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004118 struct sched_entity *se = tg->se[cpu];
Peter Zijlstraf1d239f2008-06-27 13:41:38 +02004119
Rik van Riel9722c2d2014-01-06 11:39:12 +00004120 if (!tg->parent) /* the trivial, non-cgroup case */
Peter Zijlstraf1d239f2008-06-27 13:41:38 +02004121 return wl;
4122
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004123 for_each_sched_entity(se) {
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004124 long w, W;
Peter Zijlstrabb3469a2008-06-27 13:41:27 +02004125
Paul Turner977dda72011-01-14 17:57:50 -08004126 tg = se->my_q->tg;
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004127
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004128 /*
4129 * W = @wg + \Sum rw_j
4130 */
4131 W = wg + calc_tg_weight(tg, se->my_q);
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004132
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004133 /*
4134 * w = rw_i + @wl
4135 */
4136 w = se->my_q->load.weight + wl;
Peter Zijlstra940959e2008-09-23 15:33:42 +02004137
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004138 /*
4139 * wl = S * s'_i; see (2)
4140 */
4141 if (W > 0 && w < W)
4142 wl = (w * tg->shares) / W;
Paul Turner977dda72011-01-14 17:57:50 -08004143 else
4144 wl = tg->shares;
Peter Zijlstra940959e2008-09-23 15:33:42 +02004145
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004146 /*
4147 * Per the above, wl is the new se->load.weight value; since
4148 * those are clipped to [MIN_SHARES, ...) do so now. See
4149 * calc_cfs_shares().
4150 */
Paul Turner977dda72011-01-14 17:57:50 -08004151 if (wl < MIN_SHARES)
4152 wl = MIN_SHARES;
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004153
4154 /*
4155 * wl = dw_i = S * (s'_i - s_i); see (3)
4156 */
Paul Turner977dda72011-01-14 17:57:50 -08004157 wl -= se->load.weight;
Peter Zijlstracf5f0ac2011-10-13 16:52:28 +02004158
4159 /*
4160 * Recursively apply this logic to all parent groups to compute
4161 * the final effective load change on the root group. Since
4162 * only the @tg group gets extra weight, all parent groups can
4163 * only redistribute existing shares. @wl is the shift in shares
4164 * resulting from this level per the above.
4165 */
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004166 wg = 0;
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004167 }
4168
4169 return wl;
Peter Zijlstrabb3469a2008-06-27 13:41:27 +02004170}
4171#else
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004172
Mel Gorman58d081b2013-10-07 11:29:10 +01004173static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004174{
Peter Zijlstra83378262008-06-27 13:41:37 +02004175 return wl;
Peter Zijlstrabb3469a2008-06-27 13:41:27 +02004176}
Peter Zijlstra4be9daa2008-06-27 13:41:30 +02004177
Peter Zijlstrabb3469a2008-06-27 13:41:27 +02004178#endif
4179
Michael Wang62470412013-07-04 12:55:51 +08004180static int wake_wide(struct task_struct *p)
4181{
Peter Zijlstra7d9ffa82013-07-04 12:56:46 +08004182 int factor = this_cpu_read(sd_llc_size);
Michael Wang62470412013-07-04 12:55:51 +08004183
4184 /*
4185 * Yeah, it's the switching-frequency, could means many wakee or
4186 * rapidly switch, use factor here will just help to automatically
4187 * adjust the loose-degree, so bigger node will lead to more pull.
4188 */
4189 if (p->wakee_flips > factor) {
4190 /*
4191 * wakee is somewhat hot, it needs certain amount of cpu
4192 * resource, so if waker is far more hot, prefer to leave
4193 * it alone.
4194 */
4195 if (current->wakee_flips > (factor * p->wakee_flips))
4196 return 1;
4197 }
4198
4199 return 0;
4200}
4201
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004202static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004203{
Paul Turnere37b6a72011-01-21 20:44:59 -08004204 s64 this_load, load;
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004205 int idx, this_cpu, prev_cpu;
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004206 unsigned long tl_per_task;
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004207 struct task_group *tg;
Peter Zijlstra83378262008-06-27 13:41:37 +02004208 unsigned long weight;
Mike Galbraithb3137bc2008-05-29 11:11:41 +02004209 int balanced;
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004210
Michael Wang62470412013-07-04 12:55:51 +08004211 /*
4212 * If we wake multiple tasks be careful to not bounce
4213 * ourselves around too much.
4214 */
4215 if (wake_wide(p))
4216 return 0;
4217
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004218 idx = sd->wake_idx;
4219 this_cpu = smp_processor_id();
4220 prev_cpu = task_cpu(p);
4221 load = source_load(prev_cpu, idx);
4222 this_load = target_load(this_cpu, idx);
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004223
4224 /*
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004225 * If sync wakeup then subtract the (maximum possible)
4226 * effect of the currently running task from the load
4227 * of the current CPU:
4228 */
Peter Zijlstra83378262008-06-27 13:41:37 +02004229 if (sync) {
4230 tg = task_group(current);
4231 weight = current->se.load.weight;
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004232
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004233 this_load += effective_load(tg, this_cpu, -weight, -weight);
Peter Zijlstra83378262008-06-27 13:41:37 +02004234 load += effective_load(tg, prev_cpu, 0, -weight);
4235 }
4236
4237 tg = task_group(p);
4238 weight = p->se.load.weight;
4239
Peter Zijlstra71a29aa2009-09-07 18:28:05 +02004240 /*
4241 * In low-load situations, where prev_cpu is idle and this_cpu is idle
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004242 * due to the sync cause above having dropped this_load to 0, we'll
4243 * always have an imbalance, but there's really nothing you can do
4244 * about that, so that's good too.
Peter Zijlstra71a29aa2009-09-07 18:28:05 +02004245 *
4246 * Otherwise check if either cpus are near enough in load to allow this
4247 * task to be woken on this_cpu.
4248 */
Paul Turnere37b6a72011-01-21 20:44:59 -08004249 if (this_load > 0) {
4250 s64 this_eff_load, prev_eff_load;
Peter Zijlstrae51fd5e2010-05-31 12:37:30 +02004251
4252 this_eff_load = 100;
4253 this_eff_load *= power_of(prev_cpu);
4254 this_eff_load *= this_load +
4255 effective_load(tg, this_cpu, weight, weight);
4256
4257 prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2;
4258 prev_eff_load *= power_of(this_cpu);
4259 prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight);
4260
4261 balanced = this_eff_load <= prev_eff_load;
4262 } else
4263 balanced = true;
Mike Galbraithb3137bc2008-05-29 11:11:41 +02004264
4265 /*
4266 * If the currently running task will sleep within
4267 * a reasonable amount of time then attract this newly
4268 * woken task:
4269 */
Peter Zijlstra2fb76352008-10-08 09:16:04 +02004270 if (sync && balanced)
4271 return 1;
Mike Galbraithb3137bc2008-05-29 11:11:41 +02004272
Lucas De Marchi41acab82010-03-10 23:37:45 -03004273 schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts);
Mike Galbraithb3137bc2008-05-29 11:11:41 +02004274 tl_per_task = cpu_avg_load_per_task(this_cpu);
4275
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004276 if (balanced ||
4277 (this_load <= load &&
4278 this_load + target_load(prev_cpu, idx) <= tl_per_task)) {
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004279 /*
4280 * This domain has SD_WAKE_AFFINE and
4281 * p is cache cold in this domain, and
4282 * there is no bad imbalance.
4283 */
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004284 schedstat_inc(sd, ttwu_move_affine);
Lucas De Marchi41acab82010-03-10 23:37:45 -03004285 schedstat_inc(p, se.statistics.nr_wakeups_affine);
Ingo Molnar098fb9d2008-03-16 20:36:10 +01004286
4287 return 1;
4288 }
4289 return 0;
4290}
4291
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004292/*
4293 * find_idlest_group finds and returns the least busy CPU group within the
4294 * domain.
4295 */
4296static struct sched_group *
Peter Zijlstra78e7ed52009-09-03 13:16:51 +02004297find_idlest_group(struct sched_domain *sd, struct task_struct *p,
Vincent Guittotc44f2a02013-10-18 13:52:21 +02004298 int this_cpu, int sd_flag)
Gregory Haskinse7693a32008-01-25 21:08:09 +01004299{
Andi Kleenb3bd3de2010-08-10 14:17:51 -07004300 struct sched_group *idlest = NULL, *group = sd->groups;
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004301 unsigned long min_load = ULONG_MAX, this_load = 0;
Vincent Guittotc44f2a02013-10-18 13:52:21 +02004302 int load_idx = sd->forkexec_idx;
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004303 int imbalance = 100 + (sd->imbalance_pct-100)/2;
Gregory Haskinse7693a32008-01-25 21:08:09 +01004304
Vincent Guittotc44f2a02013-10-18 13:52:21 +02004305 if (sd_flag & SD_BALANCE_WAKE)
4306 load_idx = sd->wake_idx;
4307
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004308 do {
4309 unsigned long load, avg_load;
4310 int local_group;
4311 int i;
Gregory Haskinse7693a32008-01-25 21:08:09 +01004312
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004313 /* Skip over this group if it has no CPUs allowed */
4314 if (!cpumask_intersects(sched_group_cpus(group),
Peter Zijlstrafa17b502011-06-16 12:23:22 +02004315 tsk_cpus_allowed(p)))
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004316 continue;
4317
4318 local_group = cpumask_test_cpu(this_cpu,
4319 sched_group_cpus(group));
4320
4321 /* Tally up the load of all CPUs in the group */
4322 avg_load = 0;
4323
4324 for_each_cpu(i, sched_group_cpus(group)) {
4325 /* Bias balancing toward cpus of our domain */
4326 if (local_group)
4327 load = source_load(i, load_idx);
4328 else
4329 load = target_load(i, load_idx);
4330
4331 avg_load += load;
4332 }
4333
4334 /* Adjust by relative CPU power of the group */
Peter Zijlstra9c3f75c2011-07-14 13:00:06 +02004335 avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgp->power;
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004336
4337 if (local_group) {
4338 this_load = avg_load;
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004339 } else if (avg_load < min_load) {
4340 min_load = avg_load;
4341 idlest = group;
4342 }
4343 } while (group = group->next, group != sd->groups);
4344
4345 if (!idlest || 100*this_load < imbalance*min_load)
4346 return NULL;
4347 return idlest;
4348}
4349
4350/*
4351 * find_idlest_cpu - find the idlest cpu among the cpus in group.
4352 */
4353static int
4354find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
4355{
4356 unsigned long load, min_load = ULONG_MAX;
4357 int idlest = -1;
4358 int i;
4359
4360 /* Traverse only the allowed CPUs */
Peter Zijlstrafa17b502011-06-16 12:23:22 +02004361 for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) {
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004362 load = weighted_cpuload(i);
4363
4364 if (load < min_load || (load == min_load && i == this_cpu)) {
4365 min_load = load;
4366 idlest = i;
Gregory Haskinse7693a32008-01-25 21:08:09 +01004367 }
4368 }
4369
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004370 return idlest;
4371}
Gregory Haskinse7693a32008-01-25 21:08:09 +01004372
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004373/*
Peter Zijlstraa50bde52009-11-12 15:55:28 +01004374 * Try and locate an idle CPU in the sched_domain.
4375 */
Suresh Siddha99bd5e22010-03-31 16:47:45 -07004376static int select_idle_sibling(struct task_struct *p, int target)
Peter Zijlstraa50bde52009-11-12 15:55:28 +01004377{
Suresh Siddha99bd5e22010-03-31 16:47:45 -07004378 struct sched_domain *sd;
Linus Torvalds37407ea2012-09-16 12:29:43 -07004379 struct sched_group *sg;
Mike Galbraithe0a79f52013-01-28 12:19:25 +01004380 int i = task_cpu(p);
4381
4382 if (idle_cpu(target))
4383 return target;
Peter Zijlstraa50bde52009-11-12 15:55:28 +01004384
4385 /*
Mike Galbraithe0a79f52013-01-28 12:19:25 +01004386 * If the prevous cpu is cache affine and idle, don't be stupid.
Peter Zijlstraa50bde52009-11-12 15:55:28 +01004387 */
Mike Galbraithe0a79f52013-01-28 12:19:25 +01004388 if (i != target && cpus_share_cache(i, target) && idle_cpu(i))
4389 return i;
Peter Zijlstraa50bde52009-11-12 15:55:28 +01004390
4391 /*
Linus Torvalds37407ea2012-09-16 12:29:43 -07004392 * Otherwise, iterate the domains and find an elegible idle cpu.
Peter Zijlstraa50bde52009-11-12 15:55:28 +01004393 */
Peter Zijlstra518cd622011-12-07 15:07:31 +01004394 sd = rcu_dereference(per_cpu(sd_llc, target));
Suresh Siddha77e81362011-11-17 11:08:23 -08004395 for_each_lower_domain(sd) {
Linus Torvalds37407ea2012-09-16 12:29:43 -07004396 sg = sd->groups;
4397 do {
4398 if (!cpumask_intersects(sched_group_cpus(sg),
4399 tsk_cpus_allowed(p)))
4400 goto next;
Mike Galbraith970e1782012-06-12 05:18:32 +02004401
Linus Torvalds37407ea2012-09-16 12:29:43 -07004402 for_each_cpu(i, sched_group_cpus(sg)) {
Mike Galbraithe0a79f52013-01-28 12:19:25 +01004403 if (i == target || !idle_cpu(i))
Linus Torvalds37407ea2012-09-16 12:29:43 -07004404 goto next;
4405 }
4406
4407 target = cpumask_first_and(sched_group_cpus(sg),
4408 tsk_cpus_allowed(p));
4409 goto done;
4410next:
4411 sg = sg->next;
4412 } while (sg != sd->groups);
4413 }
4414done:
Peter Zijlstraa50bde52009-11-12 15:55:28 +01004415 return target;
4416}
4417
4418/*
Morten Rasmussende91b9c2014-02-18 14:14:24 +00004419 * select_task_rq_fair: Select target runqueue for the waking task in domains
4420 * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE,
4421 * SD_BALANCE_FORK, or SD_BALANCE_EXEC.
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004422 *
Morten Rasmussende91b9c2014-02-18 14:14:24 +00004423 * Balances load by selecting the idlest cpu in the idlest group, or under
4424 * certain conditions an idle sibling cpu if the domain has SD_WAKE_AFFINE set.
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004425 *
Morten Rasmussende91b9c2014-02-18 14:14:24 +00004426 * Returns the target cpu number.
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004427 *
4428 * preempt must be disabled.
4429 */
Peter Zijlstra0017d732010-03-24 18:34:10 +01004430static int
Peter Zijlstraac66f542013-10-07 11:29:16 +01004431select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags)
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004432{
Peter Zijlstra29cd8ba2009-09-17 09:01:14 +02004433 struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004434 int cpu = smp_processor_id();
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004435 int new_cpu = cpu;
Suresh Siddha99bd5e22010-03-31 16:47:45 -07004436 int want_affine = 0;
Peter Zijlstra5158f4e2009-09-16 13:46:59 +02004437 int sync = wake_flags & WF_SYNC;
Gregory Haskinse7693a32008-01-25 21:08:09 +01004438
Peter Zijlstra29baa742012-04-23 12:11:21 +02004439 if (p->nr_cpus_allowed == 1)
Mike Galbraith76854c72011-11-22 15:18:24 +01004440 return prev_cpu;
4441
Peter Zijlstra0763a662009-09-14 19:37:39 +02004442 if (sd_flag & SD_BALANCE_WAKE) {
Peter Zijlstrafa17b502011-06-16 12:23:22 +02004443 if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004444 want_affine = 1;
4445 new_cpu = prev_cpu;
4446 }
Gregory Haskinse7693a32008-01-25 21:08:09 +01004447
Peter Zijlstradce840a2011-04-07 14:09:50 +02004448 rcu_read_lock();
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004449 for_each_domain(cpu, tmp) {
Peter Zijlstrae4f42882009-12-16 18:04:34 +01004450 if (!(tmp->flags & SD_LOAD_BALANCE))
4451 continue;
4452
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004453 /*
Suresh Siddha99bd5e22010-03-31 16:47:45 -07004454 * If both cpu and prev_cpu are part of this domain,
4455 * cpu is a valid SD_WAKE_AFFINE target.
Peter Zijlstrafe3bcfe2009-11-12 15:55:29 +01004456 */
Suresh Siddha99bd5e22010-03-31 16:47:45 -07004457 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
4458 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
4459 affine_sd = tmp;
Alex Shif03542a2012-07-26 08:55:34 +08004460 break;
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004461 }
4462
Alex Shif03542a2012-07-26 08:55:34 +08004463 if (tmp->flags & sd_flag)
Peter Zijlstra29cd8ba2009-09-17 09:01:14 +02004464 sd = tmp;
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004465 }
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004466
Mike Galbraith8b911ac2010-03-11 17:17:16 +01004467 if (affine_sd) {
Alex Shif03542a2012-07-26 08:55:34 +08004468 if (cpu != prev_cpu && wake_affine(affine_sd, p, sync))
Peter Zijlstradce840a2011-04-07 14:09:50 +02004469 prev_cpu = cpu;
4470
4471 new_cpu = select_idle_sibling(p, prev_cpu);
4472 goto unlock;
Mike Galbraith8b911ac2010-03-11 17:17:16 +01004473 }
Peter Zijlstra3b640892009-09-16 13:44:33 +02004474
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004475 while (sd) {
4476 struct sched_group *group;
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004477 int weight;
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004478
Peter Zijlstra0763a662009-09-14 19:37:39 +02004479 if (!(sd->flags & sd_flag)) {
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004480 sd = sd->child;
4481 continue;
4482 }
4483
Vincent Guittotc44f2a02013-10-18 13:52:21 +02004484 group = find_idlest_group(sd, p, cpu, sd_flag);
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004485 if (!group) {
4486 sd = sd->child;
4487 continue;
4488 }
4489
Peter Zijlstrad7c33c42009-09-11 12:45:38 +02004490 new_cpu = find_idlest_cpu(group, p, cpu);
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004491 if (new_cpu == -1 || new_cpu == cpu) {
4492 /* Now try balancing at a lower domain level of cpu */
4493 sd = sd->child;
4494 continue;
4495 }
4496
4497 /* Now try balancing at a lower domain level of new_cpu */
4498 cpu = new_cpu;
Peter Zijlstra669c55e2010-04-16 14:59:29 +02004499 weight = sd->span_weight;
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004500 sd = NULL;
4501 for_each_domain(cpu, tmp) {
Peter Zijlstra669c55e2010-04-16 14:59:29 +02004502 if (weight <= tmp->span_weight)
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004503 break;
Peter Zijlstra0763a662009-09-14 19:37:39 +02004504 if (tmp->flags & sd_flag)
Peter Zijlstraaaee1202009-09-10 13:36:25 +02004505 sd = tmp;
4506 }
4507 /* while loop will break here if sd == NULL */
Gregory Haskinse7693a32008-01-25 21:08:09 +01004508 }
Peter Zijlstradce840a2011-04-07 14:09:50 +02004509unlock:
4510 rcu_read_unlock();
Gregory Haskinse7693a32008-01-25 21:08:09 +01004511
Peter Zijlstrac88d5912009-09-10 13:50:02 +02004512 return new_cpu;
Gregory Haskinse7693a32008-01-25 21:08:09 +01004513}
Paul Turner0a74bef2012-10-04 13:18:30 +02004514
4515/*
4516 * Called immediately before a task is migrated to a new cpu; task_cpu(p) and
4517 * cfs_rq_of(p) references at time of call are still valid and identify the
4518 * previous cpu. However, the caller only guarantees p->pi_lock is held; no
4519 * other assumptions, including the state of rq->lock, should be made.
4520 */
4521static void
4522migrate_task_rq_fair(struct task_struct *p, int next_cpu)
4523{
Paul Turneraff3e492012-10-04 13:18:30 +02004524 struct sched_entity *se = &p->se;
4525 struct cfs_rq *cfs_rq = cfs_rq_of(se);
4526
4527 /*
4528 * Load tracking: accumulate removed load so that it can be processed
4529 * when we next update owning cfs_rq under rq->lock. Tasks contribute
4530 * to blocked load iff they have a positive decay-count. It can never
4531 * be negative here since on-rq tasks have decay-count == 0.
4532 */
4533 if (se->avg.decay_count) {
4534 se->avg.decay_count = -__synchronize_entity_decay(se);
Alex Shi25099402013-06-20 10:18:55 +08004535 atomic_long_add(se->avg.load_avg_contrib,
4536 &cfs_rq->removed_load);
Paul Turneraff3e492012-10-04 13:18:30 +02004537 }
Paul Turner0a74bef2012-10-04 13:18:30 +02004538}
Gregory Haskinse7693a32008-01-25 21:08:09 +01004539#endif /* CONFIG_SMP */
4540
Peter Zijlstrae52fb7c2009-01-14 12:39:19 +01004541static unsigned long
4542wakeup_gran(struct sched_entity *curr, struct sched_entity *se)
Peter Zijlstra0bbd3332008-04-19 19:44:57 +02004543{
4544 unsigned long gran = sysctl_sched_wakeup_granularity;
4545
4546 /*
Peter Zijlstrae52fb7c2009-01-14 12:39:19 +01004547 * Since its curr running now, convert the gran from real-time
4548 * to virtual-time in his units.
Mike Galbraith13814d42010-03-11 17:17:04 +01004549 *
4550 * By using 'se' instead of 'curr' we penalize light tasks, so
4551 * they get preempted easier. That is, if 'se' < 'curr' then
4552 * the resulting gran will be larger, therefore penalizing the
4553 * lighter, if otoh 'se' > 'curr' then the resulting gran will
4554 * be smaller, again penalizing the lighter task.
4555 *
4556 * This is especially important for buddies when the leftmost
4557 * task is higher priority than the buddy.
Peter Zijlstra0bbd3332008-04-19 19:44:57 +02004558 */
Shaohua Lif4ad9bd2011-04-08 12:53:09 +08004559 return calc_delta_fair(gran, se);
Peter Zijlstra0bbd3332008-04-19 19:44:57 +02004560}
4561
4562/*
Peter Zijlstra464b7522008-10-24 11:06:15 +02004563 * Should 'se' preempt 'curr'.
4564 *
4565 * |s1
4566 * |s2
4567 * |s3
4568 * g
4569 * |<--->|c
4570 *
4571 * w(c, s1) = -1
4572 * w(c, s2) = 0
4573 * w(c, s3) = 1
4574 *
4575 */
4576static int
4577wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
4578{
4579 s64 gran, vdiff = curr->vruntime - se->vruntime;
4580
4581 if (vdiff <= 0)
4582 return -1;
4583
Peter Zijlstrae52fb7c2009-01-14 12:39:19 +01004584 gran = wakeup_gran(curr, se);
Peter Zijlstra464b7522008-10-24 11:06:15 +02004585 if (vdiff > gran)
4586 return 1;
4587
4588 return 0;
4589}
4590
Peter Zijlstra02479092008-11-04 21:25:10 +01004591static void set_last_buddy(struct sched_entity *se)
4592{
Venkatesh Pallipadi69c80f32011-04-13 18:21:09 -07004593 if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
4594 return;
4595
4596 for_each_sched_entity(se)
4597 cfs_rq_of(se)->last = se;
Peter Zijlstra02479092008-11-04 21:25:10 +01004598}
4599
4600static void set_next_buddy(struct sched_entity *se)
4601{
Venkatesh Pallipadi69c80f32011-04-13 18:21:09 -07004602 if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
4603 return;
4604
4605 for_each_sched_entity(se)
4606 cfs_rq_of(se)->next = se;
Peter Zijlstra02479092008-11-04 21:25:10 +01004607}
4608
Rik van Rielac53db52011-02-01 09:51:03 -05004609static void set_skip_buddy(struct sched_entity *se)
4610{
Venkatesh Pallipadi69c80f32011-04-13 18:21:09 -07004611 for_each_sched_entity(se)
4612 cfs_rq_of(se)->skip = se;
Rik van Rielac53db52011-02-01 09:51:03 -05004613}
4614
Peter Zijlstra464b7522008-10-24 11:06:15 +02004615/*
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004616 * Preempt the current task with a newly woken task if needed:
4617 */
Peter Zijlstra5a9b86f2009-09-16 13:47:58 +02004618static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004619{
4620 struct task_struct *curr = rq->curr;
Srivatsa Vaddagiri8651a862007-10-15 17:00:12 +02004621 struct sched_entity *se = &curr->se, *pse = &p->se;
Mike Galbraith03e89e42008-12-16 08:45:30 +01004622 struct cfs_rq *cfs_rq = task_cfs_rq(curr);
Mike Galbraithf685cea2009-10-23 23:09:22 +02004623 int scale = cfs_rq->nr_running >= sched_nr_latency;
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07004624 int next_buddy_marked = 0;
Mike Galbraith03e89e42008-12-16 08:45:30 +01004625
Ingo Molnar4ae7d5c2008-03-19 01:42:00 +01004626 if (unlikely(se == pse))
4627 return;
4628
Paul Turner5238cdd2011-07-21 09:43:37 -07004629 /*
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01004630 * This is possible from callers such as move_task(), in which we
Paul Turner5238cdd2011-07-21 09:43:37 -07004631 * unconditionally check_prempt_curr() after an enqueue (which may have
4632 * lead to a throttle). This both saves work and prevents false
4633 * next-buddy nomination below.
4634 */
4635 if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
4636 return;
4637
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07004638 if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
Mike Galbraith3cb63d52009-09-11 12:01:17 +02004639 set_next_buddy(pse);
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07004640 next_buddy_marked = 1;
4641 }
Peter Zijlstra57fdc262008-09-23 15:33:45 +02004642
Bharata B Raoaec0a512008-08-28 14:42:49 +05304643 /*
4644 * We can come here with TIF_NEED_RESCHED already set from new task
4645 * wake up path.
Paul Turner5238cdd2011-07-21 09:43:37 -07004646 *
4647 * Note: this also catches the edge-case of curr being in a throttled
4648 * group (e.g. via set_curr_task), since update_curr() (in the
4649 * enqueue of curr) will have resulted in resched being set. This
4650 * prevents us from potentially nominating it as a false LAST_BUDDY
4651 * below.
Bharata B Raoaec0a512008-08-28 14:42:49 +05304652 */
4653 if (test_tsk_need_resched(curr))
4654 return;
4655
Darren Harta2f5c9a2011-02-22 13:04:33 -08004656 /* Idle tasks are by definition preempted by non-idle tasks. */
4657 if (unlikely(curr->policy == SCHED_IDLE) &&
4658 likely(p->policy != SCHED_IDLE))
4659 goto preempt;
4660
Ingo Molnar91c234b2007-10-15 17:00:18 +02004661 /*
Darren Harta2f5c9a2011-02-22 13:04:33 -08004662 * Batch and idle tasks do not preempt non-idle tasks (their preemption
4663 * is driven by the tick):
Ingo Molnar91c234b2007-10-15 17:00:18 +02004664 */
Ingo Molnar8ed92e52012-10-14 14:28:50 +02004665 if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
Ingo Molnar91c234b2007-10-15 17:00:18 +02004666 return;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004667
Peter Zijlstra3a7e73a2009-11-28 18:51:02 +01004668 find_matching_se(&se, &pse);
Paul Turner9bbd7372011-07-05 19:07:21 -07004669 update_curr(cfs_rq_of(se));
Peter Zijlstra3a7e73a2009-11-28 18:51:02 +01004670 BUG_ON(!pse);
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07004671 if (wakeup_preempt_entity(se, pse) == 1) {
4672 /*
4673 * Bias pick_next to pick the sched entity that is
4674 * triggering this preemption.
4675 */
4676 if (!next_buddy_marked)
4677 set_next_buddy(pse);
Peter Zijlstra3a7e73a2009-11-28 18:51:02 +01004678 goto preempt;
Venkatesh Pallipadi2f368252011-04-14 10:30:53 -07004679 }
Jupyung Leea65ac742009-11-17 18:51:40 +09004680
Peter Zijlstra3a7e73a2009-11-28 18:51:02 +01004681 return;
4682
4683preempt:
4684 resched_task(curr);
4685 /*
4686 * Only set the backward buddy when the current task is still
4687 * on the rq. This can happen when a wakeup gets interleaved
4688 * with schedule on the ->pre_schedule() or idle_balance()
4689 * point, either of which can * drop the rq lock.
4690 *
4691 * Also, during early boot the idle thread is in the fair class,
4692 * for obvious reasons its a bad idea to schedule back to it.
4693 */
4694 if (unlikely(!se->on_rq || curr == rq->idle))
4695 return;
4696
4697 if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
4698 set_last_buddy(se);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004699}
4700
Peter Zijlstra606dba22012-02-11 06:05:00 +01004701static struct task_struct *
4702pick_next_task_fair(struct rq *rq, struct task_struct *prev)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004703{
4704 struct cfs_rq *cfs_rq = &rq->cfs;
4705 struct sched_entity *se;
Peter Zijlstra678d5712012-02-11 06:05:00 +01004706 struct task_struct *p;
Peter Zijlstra37e117c2014-02-14 12:25:08 +01004707 int new_tasks;
Peter Zijlstra678d5712012-02-11 06:05:00 +01004708
Peter Zijlstra6e831252014-02-11 16:11:48 +01004709again:
Peter Zijlstra678d5712012-02-11 06:05:00 +01004710#ifdef CONFIG_FAIR_GROUP_SCHED
4711 if (!cfs_rq->nr_running)
Peter Zijlstra38033c32014-01-23 20:32:21 +01004712 goto idle;
Peter Zijlstra678d5712012-02-11 06:05:00 +01004713
Peter Zijlstra3f1d2a32014-02-12 10:49:30 +01004714 if (prev->sched_class != &fair_sched_class)
Peter Zijlstra678d5712012-02-11 06:05:00 +01004715 goto simple;
4716
4717 /*
4718 * Because of the set_next_buddy() in dequeue_task_fair() it is rather
4719 * likely that a next task is from the same cgroup as the current.
4720 *
4721 * Therefore attempt to avoid putting and setting the entire cgroup
4722 * hierarchy, only change the part that actually changes.
4723 */
4724
4725 do {
4726 struct sched_entity *curr = cfs_rq->curr;
4727
4728 /*
4729 * Since we got here without doing put_prev_entity() we also
4730 * have to consider cfs_rq->curr. If it is still a runnable
4731 * entity, update_curr() will update its vruntime, otherwise
4732 * forget we've ever seen it.
4733 */
4734 if (curr && curr->on_rq)
4735 update_curr(cfs_rq);
4736 else
4737 curr = NULL;
4738
4739 /*
4740 * This call to check_cfs_rq_runtime() will do the throttle and
4741 * dequeue its entity in the parent(s). Therefore the 'simple'
4742 * nr_running test will indeed be correct.
4743 */
4744 if (unlikely(check_cfs_rq_runtime(cfs_rq)))
4745 goto simple;
4746
4747 se = pick_next_entity(cfs_rq, curr);
4748 cfs_rq = group_cfs_rq(se);
4749 } while (cfs_rq);
4750
4751 p = task_of(se);
4752
4753 /*
4754 * Since we haven't yet done put_prev_entity and if the selected task
4755 * is a different task than we started out with, try and touch the
4756 * least amount of cfs_rqs.
4757 */
4758 if (prev != p) {
4759 struct sched_entity *pse = &prev->se;
4760
4761 while (!(cfs_rq = is_same_group(se, pse))) {
4762 int se_depth = se->depth;
4763 int pse_depth = pse->depth;
4764
4765 if (se_depth <= pse_depth) {
4766 put_prev_entity(cfs_rq_of(pse), pse);
4767 pse = parent_entity(pse);
4768 }
4769 if (se_depth >= pse_depth) {
4770 set_next_entity(cfs_rq_of(se), se);
4771 se = parent_entity(se);
4772 }
4773 }
4774
4775 put_prev_entity(cfs_rq, pse);
4776 set_next_entity(cfs_rq, se);
4777 }
4778
4779 if (hrtick_enabled(rq))
4780 hrtick_start_fair(rq, p);
4781
4782 return p;
4783simple:
4784 cfs_rq = &rq->cfs;
4785#endif
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004786
Tim Blechmann36ace272009-11-24 11:55:45 +01004787 if (!cfs_rq->nr_running)
Peter Zijlstra38033c32014-01-23 20:32:21 +01004788 goto idle;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004789
Peter Zijlstra3f1d2a32014-02-12 10:49:30 +01004790 put_prev_task(rq, prev);
Peter Zijlstra606dba22012-02-11 06:05:00 +01004791
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004792 do {
Peter Zijlstra678d5712012-02-11 06:05:00 +01004793 se = pick_next_entity(cfs_rq, NULL);
Peter Zijlstraf4b67552008-11-04 21:25:07 +01004794 set_next_entity(cfs_rq, se);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004795 cfs_rq = group_cfs_rq(se);
4796 } while (cfs_rq);
4797
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01004798 p = task_of(se);
Peter Zijlstra678d5712012-02-11 06:05:00 +01004799
Mike Galbraithb39e66e2011-11-22 15:20:07 +01004800 if (hrtick_enabled(rq))
4801 hrtick_start_fair(rq, p);
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01004802
4803 return p;
Peter Zijlstra38033c32014-01-23 20:32:21 +01004804
4805idle:
Kirill Tkhaie4aa3582014-03-06 13:31:55 +04004806 new_tasks = idle_balance(rq);
Peter Zijlstra37e117c2014-02-14 12:25:08 +01004807 /*
4808 * Because idle_balance() releases (and re-acquires) rq->lock, it is
4809 * possible for any higher priority task to appear. In that case we
4810 * must re-start the pick_next_entity() loop.
4811 */
Kirill Tkhaie4aa3582014-03-06 13:31:55 +04004812 if (new_tasks < 0)
Peter Zijlstra37e117c2014-02-14 12:25:08 +01004813 return RETRY_TASK;
4814
Kirill Tkhaie4aa3582014-03-06 13:31:55 +04004815 if (new_tasks > 0)
Peter Zijlstra38033c32014-01-23 20:32:21 +01004816 goto again;
Peter Zijlstra38033c32014-01-23 20:32:21 +01004817
4818 return NULL;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004819}
4820
4821/*
4822 * Account for a descheduled task:
4823 */
Ingo Molnar31ee5292007-08-09 11:16:49 +02004824static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004825{
4826 struct sched_entity *se = &prev->se;
4827 struct cfs_rq *cfs_rq;
4828
4829 for_each_sched_entity(se) {
4830 cfs_rq = cfs_rq_of(se);
Ingo Molnarab6cde22007-08-09 11:16:48 +02004831 put_prev_entity(cfs_rq, se);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004832 }
4833}
4834
Rik van Rielac53db52011-02-01 09:51:03 -05004835/*
4836 * sched_yield() is very simple
4837 *
4838 * The magic of dealing with the ->skip buddy is in pick_next_entity.
4839 */
4840static void yield_task_fair(struct rq *rq)
4841{
4842 struct task_struct *curr = rq->curr;
4843 struct cfs_rq *cfs_rq = task_cfs_rq(curr);
4844 struct sched_entity *se = &curr->se;
4845
4846 /*
4847 * Are we the only task in the tree?
4848 */
4849 if (unlikely(rq->nr_running == 1))
4850 return;
4851
4852 clear_buddies(cfs_rq, se);
4853
4854 if (curr->policy != SCHED_BATCH) {
4855 update_rq_clock(rq);
4856 /*
4857 * Update run-time statistics of the 'current'.
4858 */
4859 update_curr(cfs_rq);
Mike Galbraith916671c2011-11-22 15:21:26 +01004860 /*
4861 * Tell update_rq_clock() that we've just updated,
4862 * so we don't do microscopic update in schedule()
4863 * and double the fastpath cost.
4864 */
4865 rq->skip_clock_update = 1;
Rik van Rielac53db52011-02-01 09:51:03 -05004866 }
4867
4868 set_skip_buddy(se);
4869}
4870
Mike Galbraithd95f4122011-02-01 09:50:51 -05004871static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preempt)
4872{
4873 struct sched_entity *se = &p->se;
4874
Paul Turner5238cdd2011-07-21 09:43:37 -07004875 /* throttled hierarchies are not runnable */
4876 if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se)))
Mike Galbraithd95f4122011-02-01 09:50:51 -05004877 return false;
4878
4879 /* Tell the scheduler that we'd really like pse to run next. */
4880 set_next_buddy(se);
4881
Mike Galbraithd95f4122011-02-01 09:50:51 -05004882 yield_task_fair(rq);
4883
4884 return true;
4885}
4886
Peter Williams681f3e62007-10-24 18:23:51 +02004887#ifdef CONFIG_SMP
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02004888/**************************************************
Peter Zijlstrae9c84cb2012-07-03 13:53:26 +02004889 * Fair scheduling class load-balancing methods.
4890 *
4891 * BASICS
4892 *
4893 * The purpose of load-balancing is to achieve the same basic fairness the
4894 * per-cpu scheduler provides, namely provide a proportional amount of compute
4895 * time to each task. This is expressed in the following equation:
4896 *
4897 * W_i,n/P_i == W_j,n/P_j for all i,j (1)
4898 *
4899 * Where W_i,n is the n-th weight average for cpu i. The instantaneous weight
4900 * W_i,0 is defined as:
4901 *
4902 * W_i,0 = \Sum_j w_i,j (2)
4903 *
4904 * Where w_i,j is the weight of the j-th runnable task on cpu i. This weight
4905 * is derived from the nice value as per prio_to_weight[].
4906 *
4907 * The weight average is an exponential decay average of the instantaneous
4908 * weight:
4909 *
4910 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3)
4911 *
4912 * P_i is the cpu power (or compute capacity) of cpu i, typically it is the
4913 * fraction of 'recent' time available for SCHED_OTHER task execution. But it
4914 * can also include other factors [XXX].
4915 *
4916 * To achieve this balance we define a measure of imbalance which follows
4917 * directly from (1):
4918 *
4919 * imb_i,j = max{ avg(W/P), W_i/P_i } - min{ avg(W/P), W_j/P_j } (4)
4920 *
4921 * We them move tasks around to minimize the imbalance. In the continuous
4922 * function space it is obvious this converges, in the discrete case we get
4923 * a few fun cases generally called infeasible weight scenarios.
4924 *
4925 * [XXX expand on:
4926 * - infeasible weights;
4927 * - local vs global optima in the discrete case. ]
4928 *
4929 *
4930 * SCHED DOMAINS
4931 *
4932 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
4933 * for all i,j solution, we create a tree of cpus that follows the hardware
4934 * topology where each level pairs two lower groups (or better). This results
4935 * in O(log n) layers. Furthermore we reduce the number of cpus going up the
4936 * tree to only the first of the previous level and we decrease the frequency
4937 * of load-balance at each level inv. proportional to the number of cpus in
4938 * the groups.
4939 *
4940 * This yields:
4941 *
4942 * log_2 n 1 n
4943 * \Sum { --- * --- * 2^i } = O(n) (5)
4944 * i = 0 2^i 2^i
4945 * `- size of each group
4946 * | | `- number of cpus doing load-balance
4947 * | `- freq
4948 * `- sum over all levels
4949 *
4950 * Coupled with a limit on how many tasks we can migrate every balance pass,
4951 * this makes (5) the runtime complexity of the balancer.
4952 *
4953 * An important property here is that each CPU is still (indirectly) connected
4954 * to every other cpu in at most O(log n) steps:
4955 *
4956 * The adjacency matrix of the resulting graph is given by:
4957 *
4958 * log_2 n
4959 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6)
4960 * k = 0
4961 *
4962 * And you'll find that:
4963 *
4964 * A^(log_2 n)_i,j != 0 for all i,j (7)
4965 *
4966 * Showing there's indeed a path between every cpu in at most O(log n) steps.
4967 * The task movement gives a factor of O(m), giving a convergence complexity
4968 * of:
4969 *
4970 * O(nm log n), n := nr_cpus, m := nr_tasks (8)
4971 *
4972 *
4973 * WORK CONSERVING
4974 *
4975 * In order to avoid CPUs going idle while there's still work to do, new idle
4976 * balancing is more aggressive and has the newly idle cpu iterate up the domain
4977 * tree itself instead of relying on other CPUs to bring it work.
4978 *
4979 * This adds some complexity to both (5) and (8) but it reduces the total idle
4980 * time.
4981 *
4982 * [XXX more?]
4983 *
4984 *
4985 * CGROUPS
4986 *
4987 * Cgroups make a horror show out of (2), instead of a simple sum we get:
4988 *
4989 * s_k,i
4990 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9)
4991 * S_k
4992 *
4993 * Where
4994 *
4995 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10)
4996 *
4997 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on cpu i.
4998 *
4999 * The big problem is S_k, its a global sum needed to compute a local (W_i)
5000 * property.
5001 *
5002 * [XXX write more on how we solve this.. _after_ merging pjt's patches that
5003 * rewrite all of this once again.]
5004 */
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02005005
Hiroshi Shimamotoed387b72012-01-31 11:40:32 +09005006static unsigned long __read_mostly max_load_balance_interval = HZ/10;
5007
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01005008enum fbq_type { regular, remote, all };
5009
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005010#define LBF_ALL_PINNED 0x01
Peter Zijlstra367456c2012-02-20 21:49:09 +01005011#define LBF_NEED_BREAK 0x02
Peter Zijlstra62633222013-08-19 12:41:09 +02005012#define LBF_DST_PINNED 0x04
5013#define LBF_SOME_PINNED 0x08
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005014
5015struct lb_env {
5016 struct sched_domain *sd;
5017
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005018 struct rq *src_rq;
Prashanth Nageshappa85c1e7d2012-06-19 17:47:34 +05305019 int src_cpu;
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005020
5021 int dst_cpu;
5022 struct rq *dst_rq;
5023
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05305024 struct cpumask *dst_grpmask;
5025 int new_dst_cpu;
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005026 enum cpu_idle_type idle;
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005027 long imbalance;
Michael Wangb94031302012-07-12 16:10:13 +08005028 /* The set of CPUs under consideration for load-balancing */
5029 struct cpumask *cpus;
5030
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005031 unsigned int flags;
Peter Zijlstra367456c2012-02-20 21:49:09 +01005032
5033 unsigned int loop;
5034 unsigned int loop_break;
5035 unsigned int loop_max;
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01005036
5037 enum fbq_type fbq_type;
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005038};
5039
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005040/*
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005041 * move_task - move a task from one runqueue to another runqueue.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005042 * Both runqueues must be locked.
5043 */
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005044static void move_task(struct task_struct *p, struct lb_env *env)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005045{
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005046 deactivate_task(env->src_rq, p, 0);
5047 set_task_cpu(p, env->dst_cpu);
5048 activate_task(env->dst_rq, p, 0);
5049 check_preempt_curr(env->dst_rq, p, 0);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005050}
5051
5052/*
Peter Zijlstra029632f2011-10-25 10:00:11 +02005053 * Is this task likely cache-hot:
5054 */
5055static int
Alex Shi6037dd12014-03-12 14:51:51 +08005056task_hot(struct task_struct *p, u64 now)
Peter Zijlstra029632f2011-10-25 10:00:11 +02005057{
5058 s64 delta;
5059
5060 if (p->sched_class != &fair_sched_class)
5061 return 0;
5062
5063 if (unlikely(p->policy == SCHED_IDLE))
5064 return 0;
5065
5066 /*
5067 * Buddy candidates are cache hot:
5068 */
5069 if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
5070 (&p->se == cfs_rq_of(&p->se)->next ||
5071 &p->se == cfs_rq_of(&p->se)->last))
5072 return 1;
5073
5074 if (sysctl_sched_migration_cost == -1)
5075 return 1;
5076 if (sysctl_sched_migration_cost == 0)
5077 return 0;
5078
5079 delta = now - p->se.exec_start;
5080
5081 return delta < (s64)sysctl_sched_migration_cost;
5082}
5083
Mel Gorman3a7053b2013-10-07 11:29:00 +01005084#ifdef CONFIG_NUMA_BALANCING
5085/* Returns true if the destination node has incurred more faults */
5086static bool migrate_improves_locality(struct task_struct *p, struct lb_env *env)
5087{
5088 int src_nid, dst_nid;
5089
Rik van Rielff1df892014-01-27 17:03:41 -05005090 if (!sched_feat(NUMA_FAVOUR_HIGHER) || !p->numa_faults_memory ||
Mel Gorman3a7053b2013-10-07 11:29:00 +01005091 !(env->sd->flags & SD_NUMA)) {
5092 return false;
5093 }
5094
5095 src_nid = cpu_to_node(env->src_cpu);
5096 dst_nid = cpu_to_node(env->dst_cpu);
5097
Mel Gorman83e1d2c2013-10-07 11:29:27 +01005098 if (src_nid == dst_nid)
Mel Gorman3a7053b2013-10-07 11:29:00 +01005099 return false;
5100
Mel Gorman83e1d2c2013-10-07 11:29:27 +01005101 /* Always encourage migration to the preferred node. */
5102 if (dst_nid == p->numa_preferred_nid)
5103 return true;
5104
Rik van Riel887c2902013-10-07 11:29:31 +01005105 /* If both task and group weight improve, this move is a winner. */
5106 if (task_weight(p, dst_nid) > task_weight(p, src_nid) &&
5107 group_weight(p, dst_nid) > group_weight(p, src_nid))
Mel Gorman3a7053b2013-10-07 11:29:00 +01005108 return true;
5109
5110 return false;
5111}
Mel Gorman7a0f3082013-10-07 11:29:01 +01005112
5113
5114static bool migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
5115{
5116 int src_nid, dst_nid;
5117
5118 if (!sched_feat(NUMA) || !sched_feat(NUMA_RESIST_LOWER))
5119 return false;
5120
Rik van Rielff1df892014-01-27 17:03:41 -05005121 if (!p->numa_faults_memory || !(env->sd->flags & SD_NUMA))
Mel Gorman7a0f3082013-10-07 11:29:01 +01005122 return false;
5123
5124 src_nid = cpu_to_node(env->src_cpu);
5125 dst_nid = cpu_to_node(env->dst_cpu);
5126
Mel Gorman83e1d2c2013-10-07 11:29:27 +01005127 if (src_nid == dst_nid)
Mel Gorman7a0f3082013-10-07 11:29:01 +01005128 return false;
5129
Mel Gorman83e1d2c2013-10-07 11:29:27 +01005130 /* Migrating away from the preferred node is always bad. */
5131 if (src_nid == p->numa_preferred_nid)
5132 return true;
5133
Rik van Riel887c2902013-10-07 11:29:31 +01005134 /* If either task or group weight get worse, don't do it. */
5135 if (task_weight(p, dst_nid) < task_weight(p, src_nid) ||
5136 group_weight(p, dst_nid) < group_weight(p, src_nid))
Mel Gorman7a0f3082013-10-07 11:29:01 +01005137 return true;
5138
5139 return false;
5140}
5141
Mel Gorman3a7053b2013-10-07 11:29:00 +01005142#else
5143static inline bool migrate_improves_locality(struct task_struct *p,
5144 struct lb_env *env)
5145{
5146 return false;
5147}
Mel Gorman7a0f3082013-10-07 11:29:01 +01005148
5149static inline bool migrate_degrades_locality(struct task_struct *p,
5150 struct lb_env *env)
5151{
5152 return false;
5153}
Mel Gorman3a7053b2013-10-07 11:29:00 +01005154#endif
5155
Peter Zijlstra029632f2011-10-25 10:00:11 +02005156/*
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005157 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
5158 */
5159static
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01005160int can_migrate_task(struct task_struct *p, struct lb_env *env)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005161{
5162 int tsk_cache_hot = 0;
5163 /*
5164 * We do not migrate tasks that are:
Joonsoo Kimd3198082013-04-23 17:27:40 +09005165 * 1) throttled_lb_pair, or
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005166 * 2) cannot be migrated to this CPU due to cpus_allowed, or
Joonsoo Kimd3198082013-04-23 17:27:40 +09005167 * 3) running (obviously), or
5168 * 4) are cache-hot on their current CPU.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005169 */
Joonsoo Kimd3198082013-04-23 17:27:40 +09005170 if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
5171 return 0;
5172
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005173 if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) {
Joonsoo Kime02e60c2013-04-23 17:27:42 +09005174 int cpu;
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05305175
Lucas De Marchi41acab82010-03-10 23:37:45 -03005176 schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05305177
Peter Zijlstra62633222013-08-19 12:41:09 +02005178 env->flags |= LBF_SOME_PINNED;
5179
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05305180 /*
5181 * Remember if this task can be migrated to any other cpu in
5182 * our sched_group. We may want to revisit it if we couldn't
5183 * meet load balance goals by pulling other tasks on src_cpu.
5184 *
5185 * Also avoid computing new_dst_cpu if we have already computed
5186 * one in current iteration.
5187 */
Peter Zijlstra62633222013-08-19 12:41:09 +02005188 if (!env->dst_grpmask || (env->flags & LBF_DST_PINNED))
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05305189 return 0;
5190
Joonsoo Kime02e60c2013-04-23 17:27:42 +09005191 /* Prevent to re-select dst_cpu via env's cpus */
5192 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
5193 if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) {
Peter Zijlstra62633222013-08-19 12:41:09 +02005194 env->flags |= LBF_DST_PINNED;
Joonsoo Kime02e60c2013-04-23 17:27:42 +09005195 env->new_dst_cpu = cpu;
5196 break;
5197 }
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05305198 }
Joonsoo Kime02e60c2013-04-23 17:27:42 +09005199
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005200 return 0;
5201 }
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05305202
5203 /* Record that we found atleast one task that could run on dst_cpu */
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01005204 env->flags &= ~LBF_ALL_PINNED;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005205
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005206 if (task_running(env->src_rq, p)) {
Lucas De Marchi41acab82010-03-10 23:37:45 -03005207 schedstat_inc(p, se.statistics.nr_failed_migrations_running);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005208 return 0;
5209 }
5210
5211 /*
5212 * Aggressive migration if:
Mel Gorman3a7053b2013-10-07 11:29:00 +01005213 * 1) destination numa is preferred
5214 * 2) task is cache cold, or
5215 * 3) too many balance attempts have failed.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005216 */
Alex Shi6037dd12014-03-12 14:51:51 +08005217 tsk_cache_hot = task_hot(p, rq_clock_task(env->src_rq));
Mel Gorman7a0f3082013-10-07 11:29:01 +01005218 if (!tsk_cache_hot)
5219 tsk_cache_hot = migrate_degrades_locality(p, env);
Mel Gorman3a7053b2013-10-07 11:29:00 +01005220
5221 if (migrate_improves_locality(p, env)) {
5222#ifdef CONFIG_SCHEDSTATS
5223 if (tsk_cache_hot) {
5224 schedstat_inc(env->sd, lb_hot_gained[env->idle]);
5225 schedstat_inc(p, se.statistics.nr_forced_migrations);
5226 }
5227#endif
5228 return 1;
5229 }
5230
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005231 if (!tsk_cache_hot ||
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01005232 env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
Zhang Hang4e2dcb72013-04-10 14:04:55 +08005233
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005234 if (tsk_cache_hot) {
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01005235 schedstat_inc(env->sd, lb_hot_gained[env->idle]);
Lucas De Marchi41acab82010-03-10 23:37:45 -03005236 schedstat_inc(p, se.statistics.nr_forced_migrations);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005237 }
Zhang Hang4e2dcb72013-04-10 14:04:55 +08005238
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005239 return 1;
5240 }
5241
Zhang Hang4e2dcb72013-04-10 14:04:55 +08005242 schedstat_inc(p, se.statistics.nr_failed_migrations_hot);
5243 return 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005244}
5245
Peter Zijlstra897c3952009-12-17 17:45:42 +01005246/*
5247 * move_one_task tries to move exactly one task from busiest to this_rq, as
5248 * part of active balancing operations within "domain".
5249 * Returns 1 if successful and 0 otherwise.
5250 *
5251 * Called with both runqueues locked.
5252 */
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01005253static int move_one_task(struct lb_env *env)
Peter Zijlstra897c3952009-12-17 17:45:42 +01005254{
5255 struct task_struct *p, *n;
Peter Zijlstra897c3952009-12-17 17:45:42 +01005256
Peter Zijlstra367456c2012-02-20 21:49:09 +01005257 list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) {
Peter Zijlstra367456c2012-02-20 21:49:09 +01005258 if (!can_migrate_task(p, env))
5259 continue;
Peter Zijlstra897c3952009-12-17 17:45:42 +01005260
Peter Zijlstra367456c2012-02-20 21:49:09 +01005261 move_task(p, env);
5262 /*
5263 * Right now, this is only the second place move_task()
5264 * is called, so we can safely collect move_task()
5265 * stats here rather than inside move_task().
5266 */
5267 schedstat_inc(env->sd, lb_gained[env->idle]);
5268 return 1;
Peter Zijlstra897c3952009-12-17 17:45:42 +01005269 }
Peter Zijlstra897c3952009-12-17 17:45:42 +01005270 return 0;
5271}
5272
Peter Zijlstraeb953082012-04-17 13:38:40 +02005273static const unsigned int sched_nr_migrate_break = 32;
5274
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005275/*
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005276 * move_tasks tries to move up to imbalance weighted load from busiest to
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005277 * this_rq, as part of a balancing operation within domain "sd".
5278 * Returns 1 if successful and 0 otherwise.
5279 *
5280 * Called with both runqueues locked.
5281 */
5282static int move_tasks(struct lb_env *env)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005283{
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005284 struct list_head *tasks = &env->src_rq->cfs_tasks;
5285 struct task_struct *p;
Peter Zijlstra367456c2012-02-20 21:49:09 +01005286 unsigned long load;
5287 int pulled = 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005288
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005289 if (env->imbalance <= 0)
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005290 return 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005291
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005292 while (!list_empty(tasks)) {
5293 p = list_first_entry(tasks, struct task_struct, se.group_node);
5294
Peter Zijlstra367456c2012-02-20 21:49:09 +01005295 env->loop++;
5296 /* We've more or less seen every task there is, call it quits */
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005297 if (env->loop > env->loop_max)
Peter Zijlstra367456c2012-02-20 21:49:09 +01005298 break;
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005299
5300 /* take a breather every nr_migrate tasks */
Peter Zijlstra367456c2012-02-20 21:49:09 +01005301 if (env->loop > env->loop_break) {
Peter Zijlstraeb953082012-04-17 13:38:40 +02005302 env->loop_break += sched_nr_migrate_break;
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01005303 env->flags |= LBF_NEED_BREAK;
Peter Zijlstraee00e662009-12-17 17:25:20 +01005304 break;
Peter Zijlstraa195f002011-09-22 15:30:18 +02005305 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005306
Joonsoo Kimd3198082013-04-23 17:27:40 +09005307 if (!can_migrate_task(p, env))
Peter Zijlstra367456c2012-02-20 21:49:09 +01005308 goto next;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005309
Peter Zijlstra367456c2012-02-20 21:49:09 +01005310 load = task_h_load(p);
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005311
Peter Zijlstraeb953082012-04-17 13:38:40 +02005312 if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed)
Peter Zijlstra367456c2012-02-20 21:49:09 +01005313 goto next;
5314
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005315 if ((load / 2) > env->imbalance)
Peter Zijlstra367456c2012-02-20 21:49:09 +01005316 goto next;
5317
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005318 move_task(p, env);
Peter Zijlstraee00e662009-12-17 17:25:20 +01005319 pulled++;
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005320 env->imbalance -= load;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005321
5322#ifdef CONFIG_PREEMPT
Peter Zijlstraee00e662009-12-17 17:25:20 +01005323 /*
5324 * NEWIDLE balancing is a source of latency, so preemptible
5325 * kernels will stop after the first task is pulled to minimize
5326 * the critical section.
5327 */
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005328 if (env->idle == CPU_NEWLY_IDLE)
Peter Zijlstraee00e662009-12-17 17:25:20 +01005329 break;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005330#endif
5331
Peter Zijlstraee00e662009-12-17 17:25:20 +01005332 /*
5333 * We only want to steal up to the prescribed amount of
5334 * weighted load.
5335 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005336 if (env->imbalance <= 0)
Peter Zijlstraee00e662009-12-17 17:25:20 +01005337 break;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005338
Peter Zijlstra367456c2012-02-20 21:49:09 +01005339 continue;
5340next:
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005341 list_move_tail(&p->se.group_node, tasks);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005342 }
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005343
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005344 /*
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01005345 * Right now, this is one of only two places move_task() is called,
5346 * so we can safely collect move_task() stats here rather than
5347 * inside move_task().
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005348 */
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01005349 schedstat_add(env->sd, lb_gained[env->idle], pulled);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005350
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01005351 return pulled;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005352}
5353
Peter Zijlstra230059de2009-12-17 17:47:12 +01005354#ifdef CONFIG_FAIR_GROUP_SCHED
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005355/*
5356 * update tg->load_weight by folding this cpu's load_avg
5357 */
Paul Turner48a16752012-10-04 13:18:31 +02005358static void __update_blocked_averages_cpu(struct task_group *tg, int cpu)
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005359{
Paul Turner48a16752012-10-04 13:18:31 +02005360 struct sched_entity *se = tg->se[cpu];
5361 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005362
Paul Turner48a16752012-10-04 13:18:31 +02005363 /* throttled entities do not contribute to load */
5364 if (throttled_hierarchy(cfs_rq))
5365 return;
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005366
Paul Turneraff3e492012-10-04 13:18:30 +02005367 update_cfs_rq_blocked_load(cfs_rq, 1);
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005368
Paul Turner82958362012-10-04 13:18:31 +02005369 if (se) {
5370 update_entity_load_avg(se, 1);
5371 /*
5372 * We pivot on our runnable average having decayed to zero for
5373 * list removal. This generally implies that all our children
5374 * have also been removed (modulo rounding error or bandwidth
5375 * control); however, such cases are rare and we can fix these
5376 * at enqueue.
5377 *
5378 * TODO: fix up out-of-order children on enqueue.
5379 */
5380 if (!se->avg.runnable_avg_sum && !cfs_rq->nr_running)
5381 list_del_leaf_cfs_rq(cfs_rq);
5382 } else {
Paul Turner48a16752012-10-04 13:18:31 +02005383 struct rq *rq = rq_of(cfs_rq);
Paul Turner82958362012-10-04 13:18:31 +02005384 update_rq_runnable_avg(rq, rq->nr_running);
5385 }
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005386}
5387
Paul Turner48a16752012-10-04 13:18:31 +02005388static void update_blocked_averages(int cpu)
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005389{
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005390 struct rq *rq = cpu_rq(cpu);
Paul Turner48a16752012-10-04 13:18:31 +02005391 struct cfs_rq *cfs_rq;
5392 unsigned long flags;
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005393
Paul Turner48a16752012-10-04 13:18:31 +02005394 raw_spin_lock_irqsave(&rq->lock, flags);
5395 update_rq_clock(rq);
Peter Zijlstra9763b672011-07-13 13:09:25 +02005396 /*
5397 * Iterates the task_group tree in a bottom up fashion, see
5398 * list_add_leaf_cfs_rq() for details.
5399 */
Paul Turner64660c82011-07-21 09:43:36 -07005400 for_each_leaf_cfs_rq(rq, cfs_rq) {
Paul Turner48a16752012-10-04 13:18:31 +02005401 /*
5402 * Note: We may want to consider periodically releasing
5403 * rq->lock about these updates so that creating many task
5404 * groups does not result in continually extending hold time.
5405 */
5406 __update_blocked_averages_cpu(cfs_rq->tg, rq->cpu);
Paul Turner64660c82011-07-21 09:43:36 -07005407 }
Paul Turner48a16752012-10-04 13:18:31 +02005408
5409 raw_spin_unlock_irqrestore(&rq->lock, flags);
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005410}
5411
Peter Zijlstra9763b672011-07-13 13:09:25 +02005412/*
Vladimir Davydov68520792013-07-15 17:49:19 +04005413 * Compute the hierarchical load factor for cfs_rq and all its ascendants.
Peter Zijlstra9763b672011-07-13 13:09:25 +02005414 * This needs to be done in a top-down fashion because the load of a child
5415 * group is a fraction of its parents load.
5416 */
Vladimir Davydov68520792013-07-15 17:49:19 +04005417static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
Peter Zijlstra9763b672011-07-13 13:09:25 +02005418{
Vladimir Davydov68520792013-07-15 17:49:19 +04005419 struct rq *rq = rq_of(cfs_rq);
5420 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
Peter Zijlstraa35b6462012-08-08 21:46:40 +02005421 unsigned long now = jiffies;
Vladimir Davydov68520792013-07-15 17:49:19 +04005422 unsigned long load;
Peter Zijlstraa35b6462012-08-08 21:46:40 +02005423
Vladimir Davydov68520792013-07-15 17:49:19 +04005424 if (cfs_rq->last_h_load_update == now)
Peter Zijlstraa35b6462012-08-08 21:46:40 +02005425 return;
5426
Vladimir Davydov68520792013-07-15 17:49:19 +04005427 cfs_rq->h_load_next = NULL;
5428 for_each_sched_entity(se) {
5429 cfs_rq = cfs_rq_of(se);
5430 cfs_rq->h_load_next = se;
5431 if (cfs_rq->last_h_load_update == now)
5432 break;
5433 }
Peter Zijlstraa35b6462012-08-08 21:46:40 +02005434
Vladimir Davydov68520792013-07-15 17:49:19 +04005435 if (!se) {
Vladimir Davydov7e3115e2013-09-14 19:39:46 +04005436 cfs_rq->h_load = cfs_rq->runnable_load_avg;
Vladimir Davydov68520792013-07-15 17:49:19 +04005437 cfs_rq->last_h_load_update = now;
5438 }
5439
5440 while ((se = cfs_rq->h_load_next) != NULL) {
5441 load = cfs_rq->h_load;
5442 load = div64_ul(load * se->avg.load_avg_contrib,
5443 cfs_rq->runnable_load_avg + 1);
5444 cfs_rq = group_cfs_rq(se);
5445 cfs_rq->h_load = load;
5446 cfs_rq->last_h_load_update = now;
5447 }
Peter Zijlstra9763b672011-07-13 13:09:25 +02005448}
5449
Peter Zijlstra367456c2012-02-20 21:49:09 +01005450static unsigned long task_h_load(struct task_struct *p)
Peter Zijlstra230059de2009-12-17 17:47:12 +01005451{
Peter Zijlstra367456c2012-02-20 21:49:09 +01005452 struct cfs_rq *cfs_rq = task_cfs_rq(p);
Peter Zijlstra230059de2009-12-17 17:47:12 +01005453
Vladimir Davydov68520792013-07-15 17:49:19 +04005454 update_cfs_rq_h_load(cfs_rq);
Alex Shia003a252013-06-20 10:18:51 +08005455 return div64_ul(p->se.avg.load_avg_contrib * cfs_rq->h_load,
5456 cfs_rq->runnable_load_avg + 1);
Peter Zijlstra230059de2009-12-17 17:47:12 +01005457}
5458#else
Paul Turner48a16752012-10-04 13:18:31 +02005459static inline void update_blocked_averages(int cpu)
Peter Zijlstra9e3081c2010-11-15 15:47:02 -08005460{
5461}
5462
Peter Zijlstra367456c2012-02-20 21:49:09 +01005463static unsigned long task_h_load(struct task_struct *p)
5464{
Alex Shia003a252013-06-20 10:18:51 +08005465 return p->se.avg.load_avg_contrib;
Peter Zijlstra230059de2009-12-17 17:47:12 +01005466}
5467#endif
5468
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005469/********** Helpers for find_busiest_group ************************/
5470/*
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005471 * sg_lb_stats - stats of a sched_group required for load_balancing
5472 */
5473struct sg_lb_stats {
5474 unsigned long avg_load; /*Avg load across the CPUs of the group */
5475 unsigned long group_load; /* Total load over the CPUs of the group */
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005476 unsigned long sum_weighted_load; /* Weighted load of group's tasks */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005477 unsigned long load_per_task;
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02005478 unsigned long group_power;
Peter Zijlstra147c5fc2013-08-19 15:22:57 +02005479 unsigned int sum_nr_running; /* Nr tasks running in the group */
5480 unsigned int group_capacity;
5481 unsigned int idle_cpus;
5482 unsigned int group_weight;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005483 int group_imb; /* Is there an imbalance in the group ? */
Nikhil Raofab47622010-10-15 13:12:29 -07005484 int group_has_capacity; /* Is there extra capacity in the group? */
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01005485#ifdef CONFIG_NUMA_BALANCING
5486 unsigned int nr_numa_running;
5487 unsigned int nr_preferred_running;
5488#endif
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005489};
5490
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005491/*
5492 * sd_lb_stats - Structure to store the statistics of a sched_domain
5493 * during load balancing.
5494 */
5495struct sd_lb_stats {
5496 struct sched_group *busiest; /* Busiest group in this sd */
5497 struct sched_group *local; /* Local group in this sd */
5498 unsigned long total_load; /* Total load of all groups in sd */
5499 unsigned long total_pwr; /* Total power of all groups in sd */
5500 unsigned long avg_load; /* Average load across all groups in sd */
5501
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005502 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
Peter Zijlstra147c5fc2013-08-19 15:22:57 +02005503 struct sg_lb_stats local_stat; /* Statistics of the local group */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005504};
5505
Peter Zijlstra147c5fc2013-08-19 15:22:57 +02005506static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
5507{
5508 /*
5509 * Skimp on the clearing to avoid duplicate work. We can avoid clearing
5510 * local_stat because update_sg_lb_stats() does a full clear/assignment.
5511 * We must however clear busiest_stat::avg_load because
5512 * update_sd_pick_busiest() reads this before assignment.
5513 */
5514 *sds = (struct sd_lb_stats){
5515 .busiest = NULL,
5516 .local = NULL,
5517 .total_load = 0UL,
5518 .total_pwr = 0UL,
5519 .busiest_stat = {
5520 .avg_load = 0UL,
5521 },
5522 };
5523}
5524
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005525/**
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005526 * get_sd_load_idx - Obtain the load index for a given sched domain.
5527 * @sd: The sched_domain whose load_idx is to be obtained.
Kamalesh Babulaled1b7732013-10-13 23:06:15 +05305528 * @idle: The idle status of the CPU for whose sd load_idx is obtained.
Yacine Belkadie69f6182013-07-12 20:45:47 +02005529 *
5530 * Return: The load index.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005531 */
5532static inline int get_sd_load_idx(struct sched_domain *sd,
5533 enum cpu_idle_type idle)
5534{
5535 int load_idx;
5536
5537 switch (idle) {
5538 case CPU_NOT_IDLE:
5539 load_idx = sd->busy_idx;
5540 break;
5541
5542 case CPU_NEWLY_IDLE:
5543 load_idx = sd->newidle_idx;
5544 break;
5545 default:
5546 load_idx = sd->idle_idx;
5547 break;
5548 }
5549
5550 return load_idx;
5551}
5552
Li Zefan15f803c2013-03-05 16:07:11 +08005553static unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005554{
Nikhil Rao1399fa72011-05-18 10:09:39 -07005555 return SCHED_POWER_SCALE;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005556}
5557
5558unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
5559{
5560 return default_scale_freq_power(sd, cpu);
5561}
5562
Li Zefan15f803c2013-03-05 16:07:11 +08005563static unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005564{
Peter Zijlstra669c55e2010-04-16 14:59:29 +02005565 unsigned long weight = sd->span_weight;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005566 unsigned long smt_gain = sd->smt_gain;
5567
5568 smt_gain /= weight;
5569
5570 return smt_gain;
5571}
5572
5573unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
5574{
5575 return default_scale_smt_power(sd, cpu);
5576}
5577
Li Zefan15f803c2013-03-05 16:07:11 +08005578static unsigned long scale_rt_power(int cpu)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005579{
5580 struct rq *rq = cpu_rq(cpu);
Peter Zijlstrab654f7d2012-05-22 14:04:28 +02005581 u64 total, available, age_stamp, avg;
Peter Zijlstracadefd32014-02-27 10:40:35 +01005582 s64 delta;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005583
Peter Zijlstrab654f7d2012-05-22 14:04:28 +02005584 /*
5585 * Since we're reading these variables without serialization make sure
5586 * we read them once before doing sanity checks on them.
5587 */
5588 age_stamp = ACCESS_ONCE(rq->age_stamp);
5589 avg = ACCESS_ONCE(rq->rt_avg);
Venkatesh Pallipadiaa483802010-10-04 17:03:22 -07005590
Peter Zijlstracadefd32014-02-27 10:40:35 +01005591 delta = rq_clock(rq) - age_stamp;
5592 if (unlikely(delta < 0))
5593 delta = 0;
5594
5595 total = sched_avg_period() + delta;
Peter Zijlstrab654f7d2012-05-22 14:04:28 +02005596
5597 if (unlikely(total < avg)) {
Venkatesh Pallipadiaa483802010-10-04 17:03:22 -07005598 /* Ensures that power won't end up being negative */
5599 available = 0;
5600 } else {
Peter Zijlstrab654f7d2012-05-22 14:04:28 +02005601 available = total - avg;
Venkatesh Pallipadiaa483802010-10-04 17:03:22 -07005602 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005603
Nikhil Rao1399fa72011-05-18 10:09:39 -07005604 if (unlikely((s64)total < SCHED_POWER_SCALE))
5605 total = SCHED_POWER_SCALE;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005606
Nikhil Rao1399fa72011-05-18 10:09:39 -07005607 total >>= SCHED_POWER_SHIFT;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005608
5609 return div_u64(available, total);
5610}
5611
5612static void update_cpu_power(struct sched_domain *sd, int cpu)
5613{
Peter Zijlstra669c55e2010-04-16 14:59:29 +02005614 unsigned long weight = sd->span_weight;
Nikhil Rao1399fa72011-05-18 10:09:39 -07005615 unsigned long power = SCHED_POWER_SCALE;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005616 struct sched_group *sdg = sd->groups;
5617
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005618 if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
5619 if (sched_feat(ARCH_POWER))
5620 power *= arch_scale_smt_power(sd, cpu);
5621 else
5622 power *= default_scale_smt_power(sd, cpu);
5623
Nikhil Rao1399fa72011-05-18 10:09:39 -07005624 power >>= SCHED_POWER_SHIFT;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005625 }
5626
Peter Zijlstra9c3f75c2011-07-14 13:00:06 +02005627 sdg->sgp->power_orig = power;
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10005628
5629 if (sched_feat(ARCH_POWER))
5630 power *= arch_scale_freq_power(sd, cpu);
5631 else
5632 power *= default_scale_freq_power(sd, cpu);
5633
Nikhil Rao1399fa72011-05-18 10:09:39 -07005634 power >>= SCHED_POWER_SHIFT;
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10005635
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005636 power *= scale_rt_power(cpu);
Nikhil Rao1399fa72011-05-18 10:09:39 -07005637 power >>= SCHED_POWER_SHIFT;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005638
5639 if (!power)
5640 power = 1;
5641
Peter Zijlstrae51fd5e2010-05-31 12:37:30 +02005642 cpu_rq(cpu)->cpu_power = power;
Peter Zijlstra9c3f75c2011-07-14 13:00:06 +02005643 sdg->sgp->power = power;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005644}
5645
Peter Zijlstra029632f2011-10-25 10:00:11 +02005646void update_group_power(struct sched_domain *sd, int cpu)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005647{
5648 struct sched_domain *child = sd->child;
5649 struct sched_group *group, *sdg = sd->groups;
Peter Zijlstra863bffc2013-08-28 11:44:39 +02005650 unsigned long power, power_orig;
Vincent Guittot4ec44122011-12-12 20:21:08 +01005651 unsigned long interval;
5652
5653 interval = msecs_to_jiffies(sd->balance_interval);
5654 interval = clamp(interval, 1UL, max_load_balance_interval);
5655 sdg->sgp->next_update = jiffies + interval;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005656
5657 if (!child) {
5658 update_cpu_power(sd, cpu);
5659 return;
5660 }
5661
Peter Zijlstra863bffc2013-08-28 11:44:39 +02005662 power_orig = power = 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005663
Peter Zijlstra74a5ce22012-05-23 18:00:43 +02005664 if (child->flags & SD_OVERLAP) {
5665 /*
5666 * SD_OVERLAP domains cannot assume that child groups
5667 * span the current group.
5668 */
5669
Peter Zijlstra863bffc2013-08-28 11:44:39 +02005670 for_each_cpu(cpu, sched_group_cpus(sdg)) {
Srikar Dronamraju9abf24d2013-11-12 22:11:26 +05305671 struct sched_group_power *sgp;
5672 struct rq *rq = cpu_rq(cpu);
Peter Zijlstra863bffc2013-08-28 11:44:39 +02005673
Srikar Dronamraju9abf24d2013-11-12 22:11:26 +05305674 /*
5675 * build_sched_domains() -> init_sched_groups_power()
5676 * gets here before we've attached the domains to the
5677 * runqueues.
5678 *
5679 * Use power_of(), which is set irrespective of domains
5680 * in update_cpu_power().
5681 *
5682 * This avoids power/power_orig from being 0 and
5683 * causing divide-by-zero issues on boot.
5684 *
5685 * Runtime updates will correct power_orig.
5686 */
5687 if (unlikely(!rq->sd)) {
5688 power_orig += power_of(cpu);
5689 power += power_of(cpu);
5690 continue;
5691 }
5692
5693 sgp = rq->sd->groups->sgp;
5694 power_orig += sgp->power_orig;
5695 power += sgp->power;
Peter Zijlstra863bffc2013-08-28 11:44:39 +02005696 }
Peter Zijlstra74a5ce22012-05-23 18:00:43 +02005697 } else {
5698 /*
5699 * !SD_OVERLAP domains can assume that child groups
5700 * span the current group.
5701 */
5702
5703 group = child->groups;
5704 do {
Peter Zijlstra863bffc2013-08-28 11:44:39 +02005705 power_orig += group->sgp->power_orig;
Peter Zijlstra74a5ce22012-05-23 18:00:43 +02005706 power += group->sgp->power;
5707 group = group->next;
5708 } while (group != child->groups);
5709 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005710
Peter Zijlstra863bffc2013-08-28 11:44:39 +02005711 sdg->sgp->power_orig = power_orig;
5712 sdg->sgp->power = power;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005713}
5714
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10005715/*
5716 * Try and fix up capacity for tiny siblings, this is needed when
5717 * things like SD_ASYM_PACKING need f_b_g to select another sibling
5718 * which on its own isn't powerful enough.
5719 *
5720 * See update_sd_pick_busiest() and check_asym_packing().
5721 */
5722static inline int
5723fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
5724{
5725 /*
Nikhil Rao1399fa72011-05-18 10:09:39 -07005726 * Only siblings can have significantly less than SCHED_POWER_SCALE
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10005727 */
Peter Zijlstraa6c75f22011-04-07 14:09:52 +02005728 if (!(sd->flags & SD_SHARE_CPUPOWER))
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10005729 return 0;
5730
5731 /*
5732 * If ~90% of the cpu_power is still there, we're good.
5733 */
Peter Zijlstra9c3f75c2011-07-14 13:00:06 +02005734 if (group->sgp->power * 32 > group->sgp->power_orig * 29)
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10005735 return 1;
5736
5737 return 0;
5738}
5739
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02005740/*
5741 * Group imbalance indicates (and tries to solve) the problem where balancing
5742 * groups is inadequate due to tsk_cpus_allowed() constraints.
5743 *
5744 * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a
5745 * cpumask covering 1 cpu of the first group and 3 cpus of the second group.
5746 * Something like:
5747 *
5748 * { 0 1 2 3 } { 4 5 6 7 }
5749 * * * * *
5750 *
5751 * If we were to balance group-wise we'd place two tasks in the first group and
5752 * two tasks in the second group. Clearly this is undesired as it will overload
5753 * cpu 3 and leave one of the cpus in the second group unused.
5754 *
5755 * The current solution to this issue is detecting the skew in the first group
Peter Zijlstra62633222013-08-19 12:41:09 +02005756 * by noticing the lower domain failed to reach balance and had difficulty
5757 * moving tasks due to affinity constraints.
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02005758 *
5759 * When this is so detected; this group becomes a candidate for busiest; see
Kamalesh Babulaled1b7732013-10-13 23:06:15 +05305760 * update_sd_pick_busiest(). And calculate_imbalance() and
Peter Zijlstra62633222013-08-19 12:41:09 +02005761 * find_busiest_group() avoid some of the usual balance conditions to allow it
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02005762 * to create an effective group imbalance.
5763 *
5764 * This is a somewhat tricky proposition since the next run might not find the
5765 * group imbalance and decide the groups need to be balanced again. A most
5766 * subtle and fragile situation.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005767 */
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02005768
Peter Zijlstra62633222013-08-19 12:41:09 +02005769static inline int sg_imbalanced(struct sched_group *group)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005770{
Peter Zijlstra62633222013-08-19 12:41:09 +02005771 return group->sgp->imbalance;
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02005772}
5773
Peter Zijlstrab37d9312013-08-28 11:50:34 +02005774/*
5775 * Compute the group capacity.
5776 *
Peter Zijlstrac61037e2013-08-28 12:40:38 +02005777 * Avoid the issue where N*frac(smt_power) >= 1 creates 'phantom' cores by
5778 * first dividing out the smt factor and computing the actual number of cores
5779 * and limit power unit capacity with that.
Peter Zijlstrab37d9312013-08-28 11:50:34 +02005780 */
5781static inline int sg_capacity(struct lb_env *env, struct sched_group *group)
5782{
Peter Zijlstrac61037e2013-08-28 12:40:38 +02005783 unsigned int capacity, smt, cpus;
5784 unsigned int power, power_orig;
Peter Zijlstrab37d9312013-08-28 11:50:34 +02005785
Peter Zijlstrac61037e2013-08-28 12:40:38 +02005786 power = group->sgp->power;
5787 power_orig = group->sgp->power_orig;
5788 cpus = group->group_weight;
Peter Zijlstrab37d9312013-08-28 11:50:34 +02005789
Peter Zijlstrac61037e2013-08-28 12:40:38 +02005790 /* smt := ceil(cpus / power), assumes: 1 < smt_power < 2 */
5791 smt = DIV_ROUND_UP(SCHED_POWER_SCALE * cpus, power_orig);
5792 capacity = cpus / smt; /* cores */
5793
5794 capacity = min_t(unsigned, capacity, DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE));
Peter Zijlstrab37d9312013-08-28 11:50:34 +02005795 if (!capacity)
5796 capacity = fix_small_capacity(env->sd, group);
5797
5798 return capacity;
5799}
5800
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005801/**
5802 * update_sg_lb_stats - Update sched_group's statistics for load balancing.
5803 * @env: The load balancing environment.
5804 * @group: sched_group whose statistics are to be updated.
5805 * @load_idx: Load index of sched_domain of this_cpu for load calc.
5806 * @local_group: Does group contain this_cpu.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005807 * @sgs: variable to hold the statistics for this group.
5808 */
5809static inline void update_sg_lb_stats(struct lb_env *env,
5810 struct sched_group *group, int load_idx,
Joonsoo Kim23f0d202013-08-06 17:36:42 +09005811 int local_group, struct sg_lb_stats *sgs)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005812{
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02005813 unsigned long load;
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005814 int i;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005815
Peter Zijlstrab72ff132013-08-28 10:32:32 +02005816 memset(sgs, 0, sizeof(*sgs));
5817
Michael Wangb94031302012-07-12 16:10:13 +08005818 for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005819 struct rq *rq = cpu_rq(i);
5820
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005821 /* Bias balancing toward cpus of our domain */
Peter Zijlstra62633222013-08-19 12:41:09 +02005822 if (local_group)
Peter Zijlstra04f733b2012-05-11 00:12:02 +02005823 load = target_load(i, load_idx);
Peter Zijlstra62633222013-08-19 12:41:09 +02005824 else
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005825 load = source_load(i, load_idx);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005826
5827 sgs->group_load += load;
Kamalesh Babulal380c9072013-11-15 15:06:52 +05305828 sgs->sum_nr_running += rq->nr_running;
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01005829#ifdef CONFIG_NUMA_BALANCING
5830 sgs->nr_numa_running += rq->nr_numa_running;
5831 sgs->nr_preferred_running += rq->nr_preferred_running;
5832#endif
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005833 sgs->sum_weighted_load += weighted_cpuload(i);
Suresh Siddhaaae6d3d2010-09-17 15:02:32 -07005834 if (idle_cpu(i))
5835 sgs->idle_cpus++;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005836 }
5837
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005838 /* Adjust by relative CPU power of the group */
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02005839 sgs->group_power = group->sgp->power;
5840 sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_power;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005841
Suresh Siddhadd5feea2010-02-23 16:13:52 -08005842 if (sgs->sum_nr_running)
Peter Zijlstra38d0f772013-08-15 19:47:56 +02005843 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005844
Suresh Siddhaaae6d3d2010-09-17 15:02:32 -07005845 sgs->group_weight = group->group_weight;
Nikhil Raofab47622010-10-15 13:12:29 -07005846
Peter Zijlstrab37d9312013-08-28 11:50:34 +02005847 sgs->group_imb = sg_imbalanced(group);
5848 sgs->group_capacity = sg_capacity(env, group);
5849
Nikhil Raofab47622010-10-15 13:12:29 -07005850 if (sgs->group_capacity > sgs->sum_nr_running)
5851 sgs->group_has_capacity = 1;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005852}
5853
5854/**
Michael Neuling532cb4c2010-06-08 14:57:02 +10005855 * update_sd_pick_busiest - return 1 on busiest group
Randy Dunlapcd968912012-06-08 13:18:33 -07005856 * @env: The load balancing environment.
Michael Neuling532cb4c2010-06-08 14:57:02 +10005857 * @sds: sched_domain statistics
5858 * @sg: sched_group candidate to be checked for being the busiest
Michael Neulingb6b12292010-06-10 12:06:21 +10005859 * @sgs: sched_group statistics
Michael Neuling532cb4c2010-06-08 14:57:02 +10005860 *
5861 * Determine if @sg is a busier group than the previously selected
5862 * busiest group.
Yacine Belkadie69f6182013-07-12 20:45:47 +02005863 *
5864 * Return: %true if @sg is a busier group than the previously selected
5865 * busiest group. %false otherwise.
Michael Neuling532cb4c2010-06-08 14:57:02 +10005866 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005867static bool update_sd_pick_busiest(struct lb_env *env,
Michael Neuling532cb4c2010-06-08 14:57:02 +10005868 struct sd_lb_stats *sds,
5869 struct sched_group *sg,
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005870 struct sg_lb_stats *sgs)
Michael Neuling532cb4c2010-06-08 14:57:02 +10005871{
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005872 if (sgs->avg_load <= sds->busiest_stat.avg_load)
Michael Neuling532cb4c2010-06-08 14:57:02 +10005873 return false;
5874
5875 if (sgs->sum_nr_running > sgs->group_capacity)
5876 return true;
5877
5878 if (sgs->group_imb)
5879 return true;
5880
5881 /*
5882 * ASYM_PACKING needs to move all the work to the lowest
5883 * numbered CPUs in the group, therefore mark all groups
5884 * higher than ourself as busy.
5885 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005886 if ((env->sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running &&
5887 env->dst_cpu < group_first_cpu(sg)) {
Michael Neuling532cb4c2010-06-08 14:57:02 +10005888 if (!sds->busiest)
5889 return true;
5890
5891 if (group_first_cpu(sds->busiest) > group_first_cpu(sg))
5892 return true;
5893 }
5894
5895 return false;
5896}
5897
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01005898#ifdef CONFIG_NUMA_BALANCING
5899static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
5900{
5901 if (sgs->sum_nr_running > sgs->nr_numa_running)
5902 return regular;
5903 if (sgs->sum_nr_running > sgs->nr_preferred_running)
5904 return remote;
5905 return all;
5906}
5907
5908static inline enum fbq_type fbq_classify_rq(struct rq *rq)
5909{
5910 if (rq->nr_running > rq->nr_numa_running)
5911 return regular;
5912 if (rq->nr_running > rq->nr_preferred_running)
5913 return remote;
5914 return all;
5915}
5916#else
5917static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
5918{
5919 return all;
5920}
5921
5922static inline enum fbq_type fbq_classify_rq(struct rq *rq)
5923{
5924 return regular;
5925}
5926#endif /* CONFIG_NUMA_BALANCING */
5927
Michael Neuling532cb4c2010-06-08 14:57:02 +10005928/**
Hui Kang461819a2011-10-11 23:00:59 -04005929 * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
Randy Dunlapcd968912012-06-08 13:18:33 -07005930 * @env: The load balancing environment.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005931 * @sds: variable to hold the statistics for this sched_domain.
5932 */
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01005933static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005934{
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005935 struct sched_domain *child = env->sd->child;
5936 struct sched_group *sg = env->sd->groups;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005937 struct sg_lb_stats tmp_sgs;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005938 int load_idx, prefer_sibling = 0;
5939
5940 if (child && child->flags & SD_PREFER_SIBLING)
5941 prefer_sibling = 1;
5942
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005943 load_idx = get_sd_load_idx(env->sd, env->idle);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005944
5945 do {
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005946 struct sg_lb_stats *sgs = &tmp_sgs;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005947 int local_group;
5948
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005949 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg));
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005950 if (local_group) {
5951 sds->local = sg;
5952 sgs = &sds->local_stat;
Peter Zijlstrab72ff132013-08-28 10:32:32 +02005953
5954 if (env->idle != CPU_NEWLY_IDLE ||
5955 time_after_eq(jiffies, sg->sgp->next_update))
5956 update_group_power(env->sd, env->dst_cpu);
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005957 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005958
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005959 update_sg_lb_stats(env, sg, load_idx, local_group, sgs);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005960
Peter Zijlstrab72ff132013-08-28 10:32:32 +02005961 if (local_group)
5962 goto next_group;
5963
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005964 /*
5965 * In case the child domain prefers tasks go to siblings
Michael Neuling532cb4c2010-06-08 14:57:02 +10005966 * first, lower the sg capacity to one so that we'll try
Nikhil Rao75dd3212010-10-15 13:12:30 -07005967 * and move all the excess tasks away. We lower the capacity
5968 * of a group only if the local group has the capacity to fit
5969 * these excess tasks, i.e. nr_running < group_capacity. The
5970 * extra check prevents the case where you always pull from the
5971 * heaviest group when it is already under-utilized (possible
5972 * with a large weight task outweighs the tasks on the system).
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005973 */
Peter Zijlstrab72ff132013-08-28 10:32:32 +02005974 if (prefer_sibling && sds->local &&
5975 sds->local_stat.group_has_capacity)
Peter Zijlstra147c5fc2013-08-19 15:22:57 +02005976 sgs->group_capacity = min(sgs->group_capacity, 1U);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005977
Peter Zijlstrab72ff132013-08-28 10:32:32 +02005978 if (update_sd_pick_busiest(env, sds, sg, sgs)) {
Michael Neuling532cb4c2010-06-08 14:57:02 +10005979 sds->busiest = sg;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09005980 sds->busiest_stat = *sgs;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01005981 }
5982
Peter Zijlstrab72ff132013-08-28 10:32:32 +02005983next_group:
5984 /* Now, start updating sd_lb_stats */
5985 sds->total_load += sgs->group_load;
5986 sds->total_pwr += sgs->group_power;
5987
Michael Neuling532cb4c2010-06-08 14:57:02 +10005988 sg = sg->next;
Peter Zijlstrabd939f42012-05-02 14:20:37 +02005989 } while (sg != env->sd->groups);
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01005990
5991 if (env->sd->flags & SD_NUMA)
5992 env->fbq_type = fbq_classify_group(&sds->busiest_stat);
Michael Neuling532cb4c2010-06-08 14:57:02 +10005993}
5994
Michael Neuling532cb4c2010-06-08 14:57:02 +10005995/**
5996 * check_asym_packing - Check to see if the group is packed into the
5997 * sched doman.
5998 *
5999 * This is primarily intended to used at the sibling level. Some
6000 * cores like POWER7 prefer to use lower numbered SMT threads. In the
6001 * case of POWER7, it can move to lower SMT modes only when higher
6002 * threads are idle. When in lower SMT modes, the threads will
6003 * perform better since they share less core resources. Hence when we
6004 * have idle threads, we want them to be the higher ones.
6005 *
6006 * This packing function is run on idle threads. It checks to see if
6007 * the busiest CPU in this domain (core in the P7 case) has a higher
6008 * CPU number than the packing function is being run on. Here we are
6009 * assuming lower CPU number will be equivalent to lower a SMT thread
6010 * number.
6011 *
Yacine Belkadie69f6182013-07-12 20:45:47 +02006012 * Return: 1 when packing is required and a task should be moved to
Michael Neulingb6b12292010-06-10 12:06:21 +10006013 * this CPU. The amount of the imbalance is returned in *imbalance.
6014 *
Randy Dunlapcd968912012-06-08 13:18:33 -07006015 * @env: The load balancing environment.
Michael Neuling532cb4c2010-06-08 14:57:02 +10006016 * @sds: Statistics of the sched_domain which is to be packed
Michael Neuling532cb4c2010-06-08 14:57:02 +10006017 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006018static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)
Michael Neuling532cb4c2010-06-08 14:57:02 +10006019{
6020 int busiest_cpu;
6021
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006022 if (!(env->sd->flags & SD_ASYM_PACKING))
Michael Neuling532cb4c2010-06-08 14:57:02 +10006023 return 0;
6024
6025 if (!sds->busiest)
6026 return 0;
6027
6028 busiest_cpu = group_first_cpu(sds->busiest);
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006029 if (env->dst_cpu > busiest_cpu)
Michael Neuling532cb4c2010-06-08 14:57:02 +10006030 return 0;
6031
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006032 env->imbalance = DIV_ROUND_CLOSEST(
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006033 sds->busiest_stat.avg_load * sds->busiest_stat.group_power,
6034 SCHED_POWER_SCALE);
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006035
Michael Neuling532cb4c2010-06-08 14:57:02 +10006036 return 1;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006037}
6038
6039/**
6040 * fix_small_imbalance - Calculate the minor imbalance that exists
6041 * amongst the groups of a sched_domain, during
6042 * load balancing.
Randy Dunlapcd968912012-06-08 13:18:33 -07006043 * @env: The load balancing environment.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006044 * @sds: Statistics of the sched_domain whose imbalance is to be calculated.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006045 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006046static inline
6047void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006048{
6049 unsigned long tmp, pwr_now = 0, pwr_move = 0;
6050 unsigned int imbn = 2;
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006051 unsigned long scaled_busy_load_per_task;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006052 struct sg_lb_stats *local, *busiest;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006053
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006054 local = &sds->local_stat;
6055 busiest = &sds->busiest_stat;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006056
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006057 if (!local->sum_nr_running)
6058 local->load_per_task = cpu_avg_load_per_task(env->dst_cpu);
6059 else if (busiest->load_per_task > local->load_per_task)
6060 imbn = 1;
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006061
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006062 scaled_busy_load_per_task =
6063 (busiest->load_per_task * SCHED_POWER_SCALE) /
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006064 busiest->group_power;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006065
Vladimir Davydov3029ede2013-09-15 17:49:14 +04006066 if (busiest->avg_load + scaled_busy_load_per_task >=
6067 local->avg_load + (scaled_busy_load_per_task * imbn)) {
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006068 env->imbalance = busiest->load_per_task;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006069 return;
6070 }
6071
6072 /*
6073 * OK, we don't have enough imbalance to justify moving tasks,
6074 * however we may be able to increase total CPU power used by
6075 * moving them.
6076 */
6077
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006078 pwr_now += busiest->group_power *
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006079 min(busiest->load_per_task, busiest->avg_load);
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006080 pwr_now += local->group_power *
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006081 min(local->load_per_task, local->avg_load);
Nikhil Rao1399fa72011-05-18 10:09:39 -07006082 pwr_now /= SCHED_POWER_SCALE;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006083
6084 /* Amount of load we'd subtract */
Vincent Guittota2cd4262014-03-11 17:26:06 +01006085 if (busiest->avg_load > scaled_busy_load_per_task) {
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006086 pwr_move += busiest->group_power *
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006087 min(busiest->load_per_task,
Vincent Guittota2cd4262014-03-11 17:26:06 +01006088 busiest->avg_load - scaled_busy_load_per_task);
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006089 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006090
6091 /* Amount of load we'd add */
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006092 if (busiest->avg_load * busiest->group_power <
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006093 busiest->load_per_task * SCHED_POWER_SCALE) {
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006094 tmp = (busiest->avg_load * busiest->group_power) /
6095 local->group_power;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006096 } else {
6097 tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006098 local->group_power;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006099 }
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006100 pwr_move += local->group_power *
6101 min(local->load_per_task, local->avg_load + tmp);
Nikhil Rao1399fa72011-05-18 10:09:39 -07006102 pwr_move /= SCHED_POWER_SCALE;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006103
6104 /* Move if we gain throughput */
6105 if (pwr_move > pwr_now)
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006106 env->imbalance = busiest->load_per_task;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006107}
6108
6109/**
6110 * calculate_imbalance - Calculate the amount of imbalance present within the
6111 * groups of a given sched_domain during load balance.
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006112 * @env: load balance environment
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006113 * @sds: statistics of the sched_domain whose imbalance is to be calculated.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006114 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006115static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006116{
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006117 unsigned long max_pull, load_above_capacity = ~0UL;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006118 struct sg_lb_stats *local, *busiest;
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006119
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006120 local = &sds->local_stat;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006121 busiest = &sds->busiest_stat;
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006122
6123 if (busiest->group_imb) {
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02006124 /*
6125 * In the group_imb case we cannot rely on group-wide averages
6126 * to ensure cpu-load equilibrium, look at wider averages. XXX
6127 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006128 busiest->load_per_task =
6129 min(busiest->load_per_task, sds->avg_load);
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006130 }
6131
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006132 /*
6133 * In the presence of smp nice balancing, certain scenarios can have
6134 * max load less than avg load(as we skip the groups at or below
6135 * its cpu_power, while calculating max_load..)
6136 */
Vladimir Davydovb1885552013-09-15 17:49:13 +04006137 if (busiest->avg_load <= sds->avg_load ||
6138 local->avg_load >= sds->avg_load) {
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006139 env->imbalance = 0;
6140 return fix_small_imbalance(env, sds);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006141 }
6142
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006143 if (!busiest->group_imb) {
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006144 /*
6145 * Don't want to pull so many tasks that a group would go idle.
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02006146 * Except of course for the group_imb case, since then we might
6147 * have to drop below capacity to reach cpu-load equilibrium.
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006148 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006149 load_above_capacity =
6150 (busiest->sum_nr_running - busiest->group_capacity);
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006151
Nikhil Rao1399fa72011-05-18 10:09:39 -07006152 load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE);
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006153 load_above_capacity /= busiest->group_power;
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006154 }
6155
6156 /*
6157 * We're trying to get all the cpus to the average_load, so we don't
6158 * want to push ourselves above the average load, nor do we wish to
6159 * reduce the max loaded cpu below the average load. At the same time,
6160 * we also don't want to reduce the group load below the group capacity
6161 * (so that we can implement power-savings policies etc). Thus we look
6162 * for the minimum possible imbalance.
Suresh Siddhadd5feea2010-02-23 16:13:52 -08006163 */
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02006164 max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006165
6166 /* How much load to actually move to equalise the imbalance */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006167 env->imbalance = min(
Peter Zijlstra3ae11c92013-08-15 20:37:48 +02006168 max_pull * busiest->group_power,
6169 (sds->avg_load - local->avg_load) * local->group_power
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006170 ) / SCHED_POWER_SCALE;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006171
6172 /*
6173 * if *imbalance is less than the average load per runnable task
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006174 * there is no guarantee that any tasks will be moved so we'll have
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006175 * a think about bumping its value to force at least one task to be
6176 * moved
6177 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006178 if (env->imbalance < busiest->load_per_task)
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006179 return fix_small_imbalance(env, sds);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006180}
Nikhil Raofab47622010-10-15 13:12:29 -07006181
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006182/******* find_busiest_group() helpers end here *********************/
6183
6184/**
6185 * find_busiest_group - Returns the busiest group within the sched_domain
6186 * if there is an imbalance. If there isn't an imbalance, and
6187 * the user has opted for power-savings, it returns a group whose
6188 * CPUs can be put to idle by rebalancing those tasks elsewhere, if
6189 * such a group exists.
6190 *
6191 * Also calculates the amount of weighted load which should be moved
6192 * to restore balance.
6193 *
Randy Dunlapcd968912012-06-08 13:18:33 -07006194 * @env: The load balancing environment.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006195 *
Yacine Belkadie69f6182013-07-12 20:45:47 +02006196 * Return: - The busiest group if imbalance exists.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006197 * - If no imbalance and user has opted for power-savings balance,
6198 * return the least loaded group whose CPUs can be
6199 * put to idle by rebalancing its tasks onto our group.
6200 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006201static struct sched_group *find_busiest_group(struct lb_env *env)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006202{
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006203 struct sg_lb_stats *local, *busiest;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006204 struct sd_lb_stats sds;
6205
Peter Zijlstra147c5fc2013-08-19 15:22:57 +02006206 init_sd_lb_stats(&sds);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006207
6208 /*
6209 * Compute the various statistics relavent for load balancing at
6210 * this level.
6211 */
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006212 update_sd_lb_stats(env, &sds);
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006213 local = &sds.local_stat;
6214 busiest = &sds.busiest_stat;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006215
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006216 if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) &&
6217 check_asym_packing(env, &sds))
Michael Neuling532cb4c2010-06-08 14:57:02 +10006218 return sds.busiest;
6219
Peter Zijlstracc57aa82011-02-21 18:55:32 +01006220 /* There is no busy sibling group to pull tasks from */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006221 if (!sds.busiest || busiest->sum_nr_running == 0)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006222 goto out_balanced;
6223
Nikhil Rao1399fa72011-05-18 10:09:39 -07006224 sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr;
Ken Chenb0432d82011-04-07 17:23:22 -07006225
Peter Zijlstra866ab432011-02-21 18:56:47 +01006226 /*
6227 * If the busiest group is imbalanced the below checks don't
Peter Zijlstra30ce5da2013-08-15 20:29:29 +02006228 * work because they assume all things are equal, which typically
Peter Zijlstra866ab432011-02-21 18:56:47 +01006229 * isn't true due to cpus_allowed constraints and the like.
6230 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006231 if (busiest->group_imb)
Peter Zijlstra866ab432011-02-21 18:56:47 +01006232 goto force_balance;
6233
Peter Zijlstracc57aa82011-02-21 18:55:32 +01006234 /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006235 if (env->idle == CPU_NEWLY_IDLE && local->group_has_capacity &&
6236 !busiest->group_has_capacity)
Nikhil Raofab47622010-10-15 13:12:29 -07006237 goto force_balance;
6238
Peter Zijlstracc57aa82011-02-21 18:55:32 +01006239 /*
6240 * If the local group is more busy than the selected busiest group
6241 * don't try and pull any tasks.
6242 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006243 if (local->avg_load >= busiest->avg_load)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006244 goto out_balanced;
6245
Peter Zijlstracc57aa82011-02-21 18:55:32 +01006246 /*
6247 * Don't pull any tasks if this group is already above the domain
6248 * average load.
6249 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006250 if (local->avg_load >= sds.avg_load)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006251 goto out_balanced;
6252
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006253 if (env->idle == CPU_IDLE) {
Suresh Siddhaaae6d3d2010-09-17 15:02:32 -07006254 /*
6255 * This cpu is idle. If the busiest group load doesn't
6256 * have more tasks than the number of available cpu's and
6257 * there is no imbalance between this and busiest group
6258 * wrt to idle cpu's, it is balanced.
6259 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006260 if ((local->idle_cpus < busiest->idle_cpus) &&
6261 busiest->sum_nr_running <= busiest->group_weight)
Suresh Siddhaaae6d3d2010-09-17 15:02:32 -07006262 goto out_balanced;
Peter Zijlstrac186faf2011-02-21 18:52:53 +01006263 } else {
6264 /*
6265 * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use
6266 * imbalance_pct to be conservative.
6267 */
Joonsoo Kim56cf5152013-08-06 17:36:43 +09006268 if (100 * busiest->avg_load <=
6269 env->sd->imbalance_pct * local->avg_load)
Peter Zijlstrac186faf2011-02-21 18:52:53 +01006270 goto out_balanced;
Suresh Siddhaaae6d3d2010-09-17 15:02:32 -07006271 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006272
Nikhil Raofab47622010-10-15 13:12:29 -07006273force_balance:
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006274 /* Looks like there is an imbalance. Compute it */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006275 calculate_imbalance(env, &sds);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006276 return sds.busiest;
6277
6278out_balanced:
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006279 env->imbalance = 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006280 return NULL;
6281}
6282
6283/*
6284 * find_busiest_queue - find the busiest runqueue among the cpus in group.
6285 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006286static struct rq *find_busiest_queue(struct lb_env *env,
Michael Wangb94031302012-07-12 16:10:13 +08006287 struct sched_group *group)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006288{
6289 struct rq *busiest = NULL, *rq;
Joonsoo Kim95a79b82013-08-06 17:36:41 +09006290 unsigned long busiest_load = 0, busiest_power = 1;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006291 int i;
6292
Peter Zijlstra6906a402013-08-19 15:20:21 +02006293 for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01006294 unsigned long power, capacity, wl;
6295 enum fbq_type rt;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006296
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01006297 rq = cpu_rq(i);
6298 rt = fbq_classify_rq(rq);
6299
6300 /*
6301 * We classify groups/runqueues into three groups:
6302 * - regular: there are !numa tasks
6303 * - remote: there are numa tasks that run on the 'wrong' node
6304 * - all: there is no distinction
6305 *
6306 * In order to avoid migrating ideally placed numa tasks,
6307 * ignore those when there's better options.
6308 *
6309 * If we ignore the actual busiest queue to migrate another
6310 * task, the next balance pass can still reduce the busiest
6311 * queue by moving tasks around inside the node.
6312 *
6313 * If we cannot move enough load due to this classification
6314 * the next pass will adjust the group classification and
6315 * allow migration of more tasks.
6316 *
6317 * Both cases only affect the total convergence complexity.
6318 */
6319 if (rt > env->fbq_type)
6320 continue;
6321
6322 power = power_of(i);
6323 capacity = DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE);
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10006324 if (!capacity)
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006325 capacity = fix_small_capacity(env->sd, group);
Srivatsa Vaddagiri9d5efe02010-06-08 14:57:02 +10006326
Thomas Gleixner6e40f5b2010-02-16 16:48:56 +01006327 wl = weighted_cpuload(i);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006328
Thomas Gleixner6e40f5b2010-02-16 16:48:56 +01006329 /*
6330 * When comparing with imbalance, use weighted_cpuload()
6331 * which is not scaled with the cpu power.
6332 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006333 if (capacity && rq->nr_running == 1 && wl > env->imbalance)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006334 continue;
6335
Thomas Gleixner6e40f5b2010-02-16 16:48:56 +01006336 /*
6337 * For the load comparisons with the other cpu's, consider
6338 * the weighted_cpuload() scaled with the cpu power, so that
6339 * the load can be moved away from the cpu that is potentially
6340 * running at a lower capacity.
Joonsoo Kim95a79b82013-08-06 17:36:41 +09006341 *
6342 * Thus we're looking for max(wl_i / power_i), crosswise
6343 * multiplication to rid ourselves of the division works out
6344 * to: wl_i * power_j > wl_j * power_i; where j is our
6345 * previous maximum.
Thomas Gleixner6e40f5b2010-02-16 16:48:56 +01006346 */
Joonsoo Kim95a79b82013-08-06 17:36:41 +09006347 if (wl * busiest_power > busiest_load * power) {
6348 busiest_load = wl;
6349 busiest_power = power;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006350 busiest = rq;
6351 }
6352 }
6353
6354 return busiest;
6355}
6356
6357/*
6358 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
6359 * so long as it is large enough.
6360 */
6361#define MAX_PINNED_INTERVAL 512
6362
6363/* Working cpumask for load_balance and load_balance_newidle. */
Joonsoo Kime6252c32013-04-23 17:27:41 +09006364DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006365
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006366static int need_active_balance(struct lb_env *env)
Peter Zijlstra1af3ed32009-12-23 15:10:31 +01006367{
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006368 struct sched_domain *sd = env->sd;
6369
6370 if (env->idle == CPU_NEWLY_IDLE) {
Michael Neuling532cb4c2010-06-08 14:57:02 +10006371
6372 /*
6373 * ASYM_PACKING needs to force migrate tasks from busy but
6374 * higher numbered CPUs in order to pack all tasks in the
6375 * lowest numbered CPUs.
6376 */
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006377 if ((sd->flags & SD_ASYM_PACKING) && env->src_cpu > env->dst_cpu)
Michael Neuling532cb4c2010-06-08 14:57:02 +10006378 return 1;
Peter Zijlstra1af3ed32009-12-23 15:10:31 +01006379 }
6380
6381 return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2);
6382}
6383
Tejun Heo969c7922010-05-06 18:49:21 +02006384static int active_load_balance_cpu_stop(void *data);
6385
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006386static int should_we_balance(struct lb_env *env)
6387{
6388 struct sched_group *sg = env->sd->groups;
6389 struct cpumask *sg_cpus, *sg_mask;
6390 int cpu, balance_cpu = -1;
6391
6392 /*
6393 * In the newly idle case, we will allow all the cpu's
6394 * to do the newly idle load balance.
6395 */
6396 if (env->idle == CPU_NEWLY_IDLE)
6397 return 1;
6398
6399 sg_cpus = sched_group_cpus(sg);
6400 sg_mask = sched_group_mask(sg);
6401 /* Try to find first idle cpu */
6402 for_each_cpu_and(cpu, sg_cpus, env->cpus) {
6403 if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu))
6404 continue;
6405
6406 balance_cpu = cpu;
6407 break;
6408 }
6409
6410 if (balance_cpu == -1)
6411 balance_cpu = group_balance_cpu(sg);
6412
6413 /*
6414 * First idle cpu or the first cpu(busiest) in this sched group
6415 * is eligible for doing load balancing at this and above domains.
6416 */
Joonsoo Kimb0cff9d2013-09-10 15:54:49 +09006417 return balance_cpu == env->dst_cpu;
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006418}
6419
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006420/*
6421 * Check this_cpu to ensure it is balanced within domain. Attempt to move
6422 * tasks if there is an imbalance.
6423 */
6424static int load_balance(int this_cpu, struct rq *this_rq,
6425 struct sched_domain *sd, enum cpu_idle_type idle,
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006426 int *continue_balancing)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006427{
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306428 int ld_moved, cur_ld_moved, active_balance = 0;
Peter Zijlstra62633222013-08-19 12:41:09 +02006429 struct sched_domain *sd_parent = sd->parent;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006430 struct sched_group *group;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006431 struct rq *busiest;
6432 unsigned long flags;
Joonsoo Kime6252c32013-04-23 17:27:41 +09006433 struct cpumask *cpus = __get_cpu_var(load_balance_mask);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006434
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006435 struct lb_env env = {
6436 .sd = sd,
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01006437 .dst_cpu = this_cpu,
6438 .dst_rq = this_rq,
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306439 .dst_grpmask = sched_group_cpus(sd->groups),
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006440 .idle = idle,
Peter Zijlstraeb953082012-04-17 13:38:40 +02006441 .loop_break = sched_nr_migrate_break,
Michael Wangb94031302012-07-12 16:10:13 +08006442 .cpus = cpus,
Peter Zijlstra0ec8aa02013-10-07 11:29:33 +01006443 .fbq_type = all,
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006444 };
6445
Joonsoo Kimcfc03112013-04-23 17:27:39 +09006446 /*
6447 * For NEWLY_IDLE load_balancing, we don't need to consider
6448 * other cpus in our group
6449 */
Joonsoo Kime02e60c2013-04-23 17:27:42 +09006450 if (idle == CPU_NEWLY_IDLE)
Joonsoo Kimcfc03112013-04-23 17:27:39 +09006451 env.dst_grpmask = NULL;
Joonsoo Kimcfc03112013-04-23 17:27:39 +09006452
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006453 cpumask_copy(cpus, cpu_active_mask);
6454
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006455 schedstat_inc(sd, lb_count[idle]);
6456
6457redo:
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006458 if (!should_we_balance(&env)) {
6459 *continue_balancing = 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006460 goto out_balanced;
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006461 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006462
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006463 group = find_busiest_group(&env);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006464 if (!group) {
6465 schedstat_inc(sd, lb_nobusyg[idle]);
6466 goto out_balanced;
6467 }
6468
Michael Wangb94031302012-07-12 16:10:13 +08006469 busiest = find_busiest_queue(&env, group);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006470 if (!busiest) {
6471 schedstat_inc(sd, lb_nobusyq[idle]);
6472 goto out_balanced;
6473 }
6474
Michael Wang78feefc2012-08-06 16:41:59 +08006475 BUG_ON(busiest == env.dst_rq);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006476
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006477 schedstat_add(sd, lb_imbalance[idle], env.imbalance);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006478
6479 ld_moved = 0;
6480 if (busiest->nr_running > 1) {
6481 /*
6482 * Attempt to move tasks. If find_busiest_group has found
6483 * an imbalance but busiest->nr_running <= 1, the group is
6484 * still unbalanced. ld_moved simply stays zero, so it is
6485 * correctly treated as an imbalance.
6486 */
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006487 env.flags |= LBF_ALL_PINNED;
Peter Zijlstrac82513e2012-04-26 13:12:27 +02006488 env.src_cpu = busiest->cpu;
6489 env.src_rq = busiest;
6490 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006491
Peter Zijlstra5d6523e2012-03-10 00:07:36 +01006492more_balance:
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006493 local_irq_save(flags);
Michael Wang78feefc2012-08-06 16:41:59 +08006494 double_rq_lock(env.dst_rq, busiest);
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306495
6496 /*
6497 * cur_ld_moved - load moved in current iteration
6498 * ld_moved - cumulative load moved across iterations
6499 */
6500 cur_ld_moved = move_tasks(&env);
6501 ld_moved += cur_ld_moved;
Michael Wang78feefc2012-08-06 16:41:59 +08006502 double_rq_unlock(env.dst_rq, busiest);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006503 local_irq_restore(flags);
6504
6505 /*
6506 * some other cpu did the load balance for us.
6507 */
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306508 if (cur_ld_moved && env.dst_cpu != smp_processor_id())
6509 resched_cpu(env.dst_cpu);
6510
Joonsoo Kimf1cd0852013-04-23 17:27:37 +09006511 if (env.flags & LBF_NEED_BREAK) {
6512 env.flags &= ~LBF_NEED_BREAK;
6513 goto more_balance;
6514 }
6515
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306516 /*
6517 * Revisit (affine) tasks on src_cpu that couldn't be moved to
6518 * us and move them to an alternate dst_cpu in our sched_group
6519 * where they can run. The upper limit on how many times we
6520 * iterate on same src_cpu is dependent on number of cpus in our
6521 * sched_group.
6522 *
6523 * This changes load balance semantics a bit on who can move
6524 * load to a given_cpu. In addition to the given_cpu itself
6525 * (or a ilb_cpu acting on its behalf where given_cpu is
6526 * nohz-idle), we now have balance_cpu in a position to move
6527 * load to given_cpu. In rare situations, this may cause
6528 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding
6529 * _independently_ and at _same_ time to move some load to
6530 * given_cpu) causing exceess load to be moved to given_cpu.
6531 * This however should not happen so much in practice and
6532 * moreover subsequent load balance cycles should correct the
6533 * excess load moved.
6534 */
Peter Zijlstra62633222013-08-19 12:41:09 +02006535 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306536
Vladimir Davydov7aff2e32013-09-15 21:30:13 +04006537 /* Prevent to re-select dst_cpu via env's cpus */
6538 cpumask_clear_cpu(env.dst_cpu, env.cpus);
6539
Michael Wang78feefc2012-08-06 16:41:59 +08006540 env.dst_rq = cpu_rq(env.new_dst_cpu);
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306541 env.dst_cpu = env.new_dst_cpu;
Peter Zijlstra62633222013-08-19 12:41:09 +02006542 env.flags &= ~LBF_DST_PINNED;
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306543 env.loop = 0;
6544 env.loop_break = sched_nr_migrate_break;
Joonsoo Kime02e60c2013-04-23 17:27:42 +09006545
Srivatsa Vaddagiri88b8dac2012-06-19 17:43:15 +05306546 /*
6547 * Go back to "more_balance" rather than "redo" since we
6548 * need to continue with same src_cpu.
6549 */
6550 goto more_balance;
6551 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006552
Peter Zijlstra62633222013-08-19 12:41:09 +02006553 /*
6554 * We failed to reach balance because of affinity.
6555 */
6556 if (sd_parent) {
6557 int *group_imbalance = &sd_parent->groups->sgp->imbalance;
6558
6559 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) {
6560 *group_imbalance = 1;
6561 } else if (*group_imbalance)
6562 *group_imbalance = 0;
6563 }
6564
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006565 /* All tasks on this runqueue were pinned by CPU affinity */
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006566 if (unlikely(env.flags & LBF_ALL_PINNED)) {
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006567 cpumask_clear_cpu(cpu_of(busiest), cpus);
Prashanth Nageshappabbf18b12012-06-19 17:52:07 +05306568 if (!cpumask_empty(cpus)) {
6569 env.loop = 0;
6570 env.loop_break = sched_nr_migrate_break;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006571 goto redo;
Prashanth Nageshappabbf18b12012-06-19 17:52:07 +05306572 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006573 goto out_balanced;
6574 }
6575 }
6576
6577 if (!ld_moved) {
6578 schedstat_inc(sd, lb_failed[idle]);
Venkatesh Pallipadi58b26c42010-09-10 18:19:17 -07006579 /*
6580 * Increment the failure counter only on periodic balance.
6581 * We do not want newidle balance, which can be very
6582 * frequent, pollute the failure counter causing
6583 * excessive cache_hot migrations and active balances.
6584 */
6585 if (idle != CPU_NEWLY_IDLE)
6586 sd->nr_balance_failed++;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006587
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006588 if (need_active_balance(&env)) {
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006589 raw_spin_lock_irqsave(&busiest->lock, flags);
6590
Tejun Heo969c7922010-05-06 18:49:21 +02006591 /* don't kick the active_load_balance_cpu_stop,
6592 * if the curr task on busiest cpu can't be
6593 * moved to this_cpu
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006594 */
6595 if (!cpumask_test_cpu(this_cpu,
Peter Zijlstrafa17b502011-06-16 12:23:22 +02006596 tsk_cpus_allowed(busiest->curr))) {
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006597 raw_spin_unlock_irqrestore(&busiest->lock,
6598 flags);
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006599 env.flags |= LBF_ALL_PINNED;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006600 goto out_one_pinned;
6601 }
6602
Tejun Heo969c7922010-05-06 18:49:21 +02006603 /*
6604 * ->active_balance synchronizes accesses to
6605 * ->active_balance_work. Once set, it's cleared
6606 * only after active load balance is finished.
6607 */
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006608 if (!busiest->active_balance) {
6609 busiest->active_balance = 1;
6610 busiest->push_cpu = this_cpu;
6611 active_balance = 1;
6612 }
6613 raw_spin_unlock_irqrestore(&busiest->lock, flags);
Tejun Heo969c7922010-05-06 18:49:21 +02006614
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006615 if (active_balance) {
Tejun Heo969c7922010-05-06 18:49:21 +02006616 stop_one_cpu_nowait(cpu_of(busiest),
6617 active_load_balance_cpu_stop, busiest,
6618 &busiest->active_balance_work);
Peter Zijlstrabd939f42012-05-02 14:20:37 +02006619 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006620
6621 /*
6622 * We've kicked active balancing, reset the failure
6623 * counter.
6624 */
6625 sd->nr_balance_failed = sd->cache_nice_tries+1;
6626 }
6627 } else
6628 sd->nr_balance_failed = 0;
6629
6630 if (likely(!active_balance)) {
6631 /* We were unbalanced, so reset the balancing interval */
6632 sd->balance_interval = sd->min_interval;
6633 } else {
6634 /*
6635 * If we've begun active balancing, start to back off. This
6636 * case may not be covered by the all_pinned logic if there
6637 * is only 1 task on the busy runqueue (because we don't call
6638 * move_tasks).
6639 */
6640 if (sd->balance_interval < sd->max_interval)
6641 sd->balance_interval *= 2;
6642 }
6643
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006644 goto out;
6645
6646out_balanced:
6647 schedstat_inc(sd, lb_balanced[idle]);
6648
6649 sd->nr_balance_failed = 0;
6650
6651out_one_pinned:
6652 /* tune up the balancing interval */
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006653 if (((env.flags & LBF_ALL_PINNED) &&
Peter Zijlstra5b54b562011-09-22 15:23:13 +02006654 sd->balance_interval < MAX_PINNED_INTERVAL) ||
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006655 (sd->balance_interval < sd->max_interval))
6656 sd->balance_interval *= 2;
6657
Venkatesh Pallipadi46e49b32011-02-14 14:38:50 -08006658 ld_moved = 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006659out:
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006660 return ld_moved;
6661}
6662
6663/*
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006664 * idle_balance is called by schedule() if this_cpu is about to become
6665 * idle. Attempts to pull tasks from other CPUs.
6666 */
Peter Zijlstra6e831252014-02-11 16:11:48 +01006667static int idle_balance(struct rq *this_rq)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006668{
6669 struct sched_domain *sd;
6670 int pulled_task = 0;
6671 unsigned long next_balance = jiffies + HZ;
Jason Low9bd721c2013-09-13 11:26:52 -07006672 u64 curr_cost = 0;
Daniel Lezcanob4f2ab432014-01-17 10:04:01 +01006673 int this_cpu = this_rq->cpu;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006674
Peter Zijlstra6e831252014-02-11 16:11:48 +01006675 idle_enter_fair(this_rq);
Jason Low0e5b5332014-04-28 15:45:54 -07006676
Peter Zijlstra6e831252014-02-11 16:11:48 +01006677 /*
6678 * We must set idle_stamp _before_ calling idle_balance(), such that we
6679 * measure the duration of idle_balance() as idle time.
6680 */
6681 this_rq->idle_stamp = rq_clock(this_rq);
6682
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006683 if (this_rq->avg_idle < sysctl_sched_migration_cost)
Peter Zijlstra6e831252014-02-11 16:11:48 +01006684 goto out;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006685
Peter Zijlstraf492e122009-12-23 15:29:42 +01006686 /*
6687 * Drop the rq->lock, but keep IRQ/preempt disabled.
6688 */
6689 raw_spin_unlock(&this_rq->lock);
6690
Paul Turner48a16752012-10-04 13:18:31 +02006691 update_blocked_averages(this_cpu);
Peter Zijlstradce840a2011-04-07 14:09:50 +02006692 rcu_read_lock();
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006693 for_each_domain(this_cpu, sd) {
6694 unsigned long interval;
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006695 int continue_balancing = 1;
Jason Low9bd721c2013-09-13 11:26:52 -07006696 u64 t0, domain_cost;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006697
6698 if (!(sd->flags & SD_LOAD_BALANCE))
6699 continue;
6700
Jason Low9bd721c2013-09-13 11:26:52 -07006701 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)
6702 break;
6703
Peter Zijlstraf492e122009-12-23 15:29:42 +01006704 if (sd->flags & SD_BALANCE_NEWIDLE) {
Jason Low9bd721c2013-09-13 11:26:52 -07006705 t0 = sched_clock_cpu(this_cpu);
6706
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006707 /* If we've pulled tasks over stop searching: */
Peter Zijlstraf492e122009-12-23 15:29:42 +01006708 pulled_task = load_balance(this_cpu, this_rq,
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006709 sd, CPU_NEWLY_IDLE,
6710 &continue_balancing);
Jason Low9bd721c2013-09-13 11:26:52 -07006711
6712 domain_cost = sched_clock_cpu(this_cpu) - t0;
6713 if (domain_cost > sd->max_newidle_lb_cost)
6714 sd->max_newidle_lb_cost = domain_cost;
6715
6716 curr_cost += domain_cost;
Peter Zijlstraf492e122009-12-23 15:29:42 +01006717 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006718
6719 interval = msecs_to_jiffies(sd->balance_interval);
6720 if (time_after(next_balance, sd->last_balance + interval))
6721 next_balance = sd->last_balance + interval;
Daniel Lezcano3c4017c2014-01-17 10:04:03 +01006722 if (pulled_task)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006723 break;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006724 }
Peter Zijlstradce840a2011-04-07 14:09:50 +02006725 rcu_read_unlock();
Peter Zijlstraf492e122009-12-23 15:29:42 +01006726
6727 raw_spin_lock(&this_rq->lock);
6728
Jason Low0e5b5332014-04-28 15:45:54 -07006729 if (curr_cost > this_rq->max_idle_balance_cost)
6730 this_rq->max_idle_balance_cost = curr_cost;
6731
Daniel Lezcanoe5fc6612014-01-17 10:04:02 +01006732 /*
Jason Low0e5b5332014-04-28 15:45:54 -07006733 * While browsing the domains, we released the rq lock, a task could
6734 * have been enqueued in the meantime. Since we're not going idle,
6735 * pretend we pulled a task.
Daniel Lezcanoe5fc6612014-01-17 10:04:02 +01006736 */
Jason Low0e5b5332014-04-28 15:45:54 -07006737 if (this_rq->cfs.h_nr_running && !pulled_task)
Peter Zijlstra6e831252014-02-11 16:11:48 +01006738 pulled_task = 1;
Daniel Lezcanoe5fc6612014-01-17 10:04:02 +01006739
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006740 if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
6741 /*
6742 * We are going idle. next_balance may be set based on
6743 * a busy processor. So reset next_balance.
6744 */
6745 this_rq->next_balance = next_balance;
6746 }
Jason Low9bd721c2013-09-13 11:26:52 -07006747
Peter Zijlstra6e831252014-02-11 16:11:48 +01006748out:
Kirill Tkhaie4aa3582014-03-06 13:31:55 +04006749 /* Is there a task of a high priority class? */
Kirill Tkhai46383642014-03-15 02:15:07 +04006750 if (this_rq->nr_running != this_rq->cfs.h_nr_running)
Kirill Tkhaie4aa3582014-03-06 13:31:55 +04006751 pulled_task = -1;
6752
6753 if (pulled_task) {
6754 idle_exit_fair(this_rq);
Peter Zijlstra6e831252014-02-11 16:11:48 +01006755 this_rq->idle_stamp = 0;
Kirill Tkhaie4aa3582014-03-06 13:31:55 +04006756 }
Peter Zijlstra6e831252014-02-11 16:11:48 +01006757
Daniel Lezcano3c4017c2014-01-17 10:04:03 +01006758 return pulled_task;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006759}
6760
6761/*
Tejun Heo969c7922010-05-06 18:49:21 +02006762 * active_load_balance_cpu_stop is run by cpu stopper. It pushes
6763 * running tasks off the busiest CPU onto idle CPUs. It requires at
6764 * least 1 task to be running on each physical CPU where possible, and
6765 * avoids physical / logical imbalances.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006766 */
Tejun Heo969c7922010-05-06 18:49:21 +02006767static int active_load_balance_cpu_stop(void *data)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006768{
Tejun Heo969c7922010-05-06 18:49:21 +02006769 struct rq *busiest_rq = data;
6770 int busiest_cpu = cpu_of(busiest_rq);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006771 int target_cpu = busiest_rq->push_cpu;
Tejun Heo969c7922010-05-06 18:49:21 +02006772 struct rq *target_rq = cpu_rq(target_cpu);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006773 struct sched_domain *sd;
Tejun Heo969c7922010-05-06 18:49:21 +02006774
6775 raw_spin_lock_irq(&busiest_rq->lock);
6776
6777 /* make sure the requested cpu hasn't gone down in the meantime */
6778 if (unlikely(busiest_cpu != smp_processor_id() ||
6779 !busiest_rq->active_balance))
6780 goto out_unlock;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006781
6782 /* Is there any task to move? */
6783 if (busiest_rq->nr_running <= 1)
Tejun Heo969c7922010-05-06 18:49:21 +02006784 goto out_unlock;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006785
6786 /*
6787 * This condition is "impossible", if it occurs
6788 * we need to fix it. Originally reported by
6789 * Bjorn Helgaas on a 128-cpu setup.
6790 */
6791 BUG_ON(busiest_rq == target_rq);
6792
6793 /* move a task from busiest_rq to target_rq */
6794 double_lock_balance(busiest_rq, target_rq);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006795
6796 /* Search for an sd spanning us and the target CPU. */
Peter Zijlstradce840a2011-04-07 14:09:50 +02006797 rcu_read_lock();
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006798 for_each_domain(target_cpu, sd) {
6799 if ((sd->flags & SD_LOAD_BALANCE) &&
6800 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
6801 break;
6802 }
6803
6804 if (likely(sd)) {
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006805 struct lb_env env = {
6806 .sd = sd,
Peter Zijlstraddcdf6e2012-02-22 19:27:40 +01006807 .dst_cpu = target_cpu,
6808 .dst_rq = target_rq,
6809 .src_cpu = busiest_rq->cpu,
6810 .src_rq = busiest_rq,
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006811 .idle = CPU_IDLE,
6812 };
6813
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006814 schedstat_inc(sd, alb_count);
6815
Peter Zijlstra8e45cb52012-02-22 12:47:19 +01006816 if (move_one_task(&env))
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006817 schedstat_inc(sd, alb_pushed);
6818 else
6819 schedstat_inc(sd, alb_failed);
6820 }
Peter Zijlstradce840a2011-04-07 14:09:50 +02006821 rcu_read_unlock();
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006822 double_unlock_balance(busiest_rq, target_rq);
Tejun Heo969c7922010-05-06 18:49:21 +02006823out_unlock:
6824 busiest_rq->active_balance = 0;
6825 raw_spin_unlock_irq(&busiest_rq->lock);
6826 return 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006827}
6828
Mike Galbraithd987fc72011-12-05 10:01:47 +01006829static inline int on_null_domain(struct rq *rq)
6830{
6831 return unlikely(!rcu_dereference_sched(rq->sd));
6832}
6833
Frederic Weisbecker3451d022011-08-10 23:21:01 +02006834#ifdef CONFIG_NO_HZ_COMMON
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006835/*
6836 * idle load balancing details
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006837 * - When one of the busy CPUs notice that there may be an idle rebalancing
6838 * needed, they will kick the idle load balancer, which then does idle
6839 * load balancing for all the idle CPUs.
6840 */
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006841static struct {
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006842 cpumask_var_t idle_cpus_mask;
Suresh Siddha0b005cf2011-12-01 17:07:34 -08006843 atomic_t nr_cpus;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006844 unsigned long next_balance; /* in jiffy units */
6845} nohz ____cacheline_aligned;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006846
Daniel Lezcano3dd03372014-01-06 12:34:41 +01006847static inline int find_new_ilb(void)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006848{
Suresh Siddha0b005cf2011-12-01 17:07:34 -08006849 int ilb = cpumask_first(nohz.idle_cpus_mask);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006850
Suresh Siddha786d6dc2011-12-01 17:07:35 -08006851 if (ilb < nr_cpu_ids && idle_cpu(ilb))
6852 return ilb;
6853
6854 return nr_cpu_ids;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006855}
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006856
6857/*
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006858 * Kick a CPU to do the nohz balancing, if it is time for it. We pick the
6859 * nohz_load_balancer CPU (if there is one) otherwise fallback to any idle
6860 * CPU (if there is one).
6861 */
Daniel Lezcano0aeeeeb2014-01-06 12:34:42 +01006862static void nohz_balancer_kick(void)
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006863{
6864 int ilb_cpu;
6865
6866 nohz.next_balance++;
6867
Daniel Lezcano3dd03372014-01-06 12:34:41 +01006868 ilb_cpu = find_new_ilb();
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006869
Suresh Siddha0b005cf2011-12-01 17:07:34 -08006870 if (ilb_cpu >= nr_cpu_ids)
6871 return;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006872
Suresh Siddhacd490c52011-12-06 11:26:34 -08006873 if (test_and_set_bit(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu)))
Suresh Siddha1c792db2011-12-01 17:07:32 -08006874 return;
6875 /*
6876 * Use smp_send_reschedule() instead of resched_cpu().
6877 * This way we generate a sched IPI on the target cpu which
6878 * is idle. And the softirq performing nohz idle load balance
6879 * will be run before returning from the IPI.
6880 */
6881 smp_send_reschedule(ilb_cpu);
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006882 return;
6883}
6884
Alex Shic1cc0172012-09-10 15:10:58 +08006885static inline void nohz_balance_exit_idle(int cpu)
Suresh Siddha71325962012-01-19 18:28:57 -08006886{
6887 if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) {
Mike Galbraithd987fc72011-12-05 10:01:47 +01006888 /*
6889 * Completely isolated CPUs don't ever set, so we must test.
6890 */
6891 if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) {
6892 cpumask_clear_cpu(cpu, nohz.idle_cpus_mask);
6893 atomic_dec(&nohz.nr_cpus);
6894 }
Suresh Siddha71325962012-01-19 18:28:57 -08006895 clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
6896 }
6897}
6898
Suresh Siddha69e1e812011-12-01 17:07:33 -08006899static inline void set_cpu_sd_state_busy(void)
6900{
6901 struct sched_domain *sd;
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05306902 int cpu = smp_processor_id();
Suresh Siddha69e1e812011-12-01 17:07:33 -08006903
Suresh Siddha69e1e812011-12-01 17:07:33 -08006904 rcu_read_lock();
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05306905 sd = rcu_dereference(per_cpu(sd_busy, cpu));
Vincent Guittot25f55d92013-04-23 16:59:02 +02006906
6907 if (!sd || !sd->nohz_idle)
6908 goto unlock;
6909 sd->nohz_idle = 0;
6910
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05306911 atomic_inc(&sd->groups->sgp->nr_busy_cpus);
Vincent Guittot25f55d92013-04-23 16:59:02 +02006912unlock:
Suresh Siddha69e1e812011-12-01 17:07:33 -08006913 rcu_read_unlock();
6914}
6915
6916void set_cpu_sd_state_idle(void)
6917{
6918 struct sched_domain *sd;
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05306919 int cpu = smp_processor_id();
Suresh Siddha69e1e812011-12-01 17:07:33 -08006920
Suresh Siddha69e1e812011-12-01 17:07:33 -08006921 rcu_read_lock();
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05306922 sd = rcu_dereference(per_cpu(sd_busy, cpu));
Vincent Guittot25f55d92013-04-23 16:59:02 +02006923
6924 if (!sd || sd->nohz_idle)
6925 goto unlock;
6926 sd->nohz_idle = 1;
6927
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05306928 atomic_dec(&sd->groups->sgp->nr_busy_cpus);
Vincent Guittot25f55d92013-04-23 16:59:02 +02006929unlock:
Suresh Siddha69e1e812011-12-01 17:07:33 -08006930 rcu_read_unlock();
6931}
6932
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07006933/*
Alex Shic1cc0172012-09-10 15:10:58 +08006934 * This routine will record that the cpu is going idle with tick stopped.
Suresh Siddha0b005cf2011-12-01 17:07:34 -08006935 * This info will be used in performing idle load balancing in the future.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006936 */
Alex Shic1cc0172012-09-10 15:10:58 +08006937void nohz_balance_enter_idle(int cpu)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006938{
Suresh Siddha71325962012-01-19 18:28:57 -08006939 /*
6940 * If this cpu is going down, then nothing needs to be done.
6941 */
6942 if (!cpu_active(cpu))
6943 return;
6944
Alex Shic1cc0172012-09-10 15:10:58 +08006945 if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))
6946 return;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006947
Mike Galbraithd987fc72011-12-05 10:01:47 +01006948 /*
6949 * If we're a completely isolated CPU, we don't play.
6950 */
6951 if (on_null_domain(cpu_rq(cpu)))
6952 return;
6953
Alex Shic1cc0172012-09-10 15:10:58 +08006954 cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
6955 atomic_inc(&nohz.nr_cpus);
6956 set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006957}
Suresh Siddha71325962012-01-19 18:28:57 -08006958
Paul Gortmaker0db06282013-06-19 14:53:51 -04006959static int sched_ilb_notifier(struct notifier_block *nfb,
Suresh Siddha71325962012-01-19 18:28:57 -08006960 unsigned long action, void *hcpu)
6961{
6962 switch (action & ~CPU_TASKS_FROZEN) {
6963 case CPU_DYING:
Alex Shic1cc0172012-09-10 15:10:58 +08006964 nohz_balance_exit_idle(smp_processor_id());
Suresh Siddha71325962012-01-19 18:28:57 -08006965 return NOTIFY_OK;
6966 default:
6967 return NOTIFY_DONE;
6968 }
6969}
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006970#endif
6971
6972static DEFINE_SPINLOCK(balancing);
6973
Peter Zijlstra49c022e2011-04-05 10:14:25 +02006974/*
6975 * Scale the max load_balance interval with the number of CPUs in the system.
6976 * This trades load-balance latency on larger machines for less cross talk.
6977 */
Peter Zijlstra029632f2011-10-25 10:00:11 +02006978void update_max_interval(void)
Peter Zijlstra49c022e2011-04-05 10:14:25 +02006979{
6980 max_load_balance_interval = HZ*num_online_cpus()/10;
6981}
6982
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006983/*
6984 * It checks each scheduling domain to see if it is due to be balanced,
6985 * and initiates a balancing operation if so.
6986 *
Libinb9b08532013-04-01 19:14:01 +08006987 * Balancing parameters are set up in init_sched_domains.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006988 */
Daniel Lezcanof7ed0a82014-01-06 12:34:43 +01006989static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006990{
Joonsoo Kim23f0d202013-08-06 17:36:42 +09006991 int continue_balancing = 1;
Daniel Lezcanof7ed0a82014-01-06 12:34:43 +01006992 int cpu = rq->cpu;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006993 unsigned long interval;
Peter Zijlstra04f733b2012-05-11 00:12:02 +02006994 struct sched_domain *sd;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01006995 /* Earliest time when we have to do rebalance again */
6996 unsigned long next_balance = jiffies + 60*HZ;
6997 int update_next_balance = 0;
Jason Lowf48627e2013-09-13 11:26:53 -07006998 int need_serialize, need_decay = 0;
6999 u64 max_cost = 0;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007000
Paul Turner48a16752012-10-04 13:18:31 +02007001 update_blocked_averages(cpu);
Peter Zijlstra2069dd72010-11-15 15:47:00 -08007002
Peter Zijlstradce840a2011-04-07 14:09:50 +02007003 rcu_read_lock();
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007004 for_each_domain(cpu, sd) {
Jason Lowf48627e2013-09-13 11:26:53 -07007005 /*
7006 * Decay the newidle max times here because this is a regular
7007 * visit to all the domains. Decay ~1% per second.
7008 */
7009 if (time_after(jiffies, sd->next_decay_max_lb_cost)) {
7010 sd->max_newidle_lb_cost =
7011 (sd->max_newidle_lb_cost * 253) / 256;
7012 sd->next_decay_max_lb_cost = jiffies + HZ;
7013 need_decay = 1;
7014 }
7015 max_cost += sd->max_newidle_lb_cost;
7016
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007017 if (!(sd->flags & SD_LOAD_BALANCE))
7018 continue;
7019
Jason Lowf48627e2013-09-13 11:26:53 -07007020 /*
7021 * Stop the load balance at this level. There is another
7022 * CPU in our sched group which is doing load balancing more
7023 * actively.
7024 */
7025 if (!continue_balancing) {
7026 if (need_decay)
7027 continue;
7028 break;
7029 }
7030
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007031 interval = sd->balance_interval;
7032 if (idle != CPU_IDLE)
7033 interval *= sd->busy_factor;
7034
7035 /* scale ms to jiffies */
7036 interval = msecs_to_jiffies(interval);
Peter Zijlstra49c022e2011-04-05 10:14:25 +02007037 interval = clamp(interval, 1UL, max_load_balance_interval);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007038
7039 need_serialize = sd->flags & SD_SERIALIZE;
7040
7041 if (need_serialize) {
7042 if (!spin_trylock(&balancing))
7043 goto out;
7044 }
7045
7046 if (time_after_eq(jiffies, sd->last_balance + interval)) {
Joonsoo Kim23f0d202013-08-06 17:36:42 +09007047 if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007048 /*
Peter Zijlstra62633222013-08-19 12:41:09 +02007049 * The LBF_DST_PINNED logic could have changed
Joonsoo Kimde5eb2d2013-04-23 17:27:38 +09007050 * env->dst_cpu, so we can't know our idle
7051 * state even if we migrated tasks. Update it.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007052 */
Joonsoo Kimde5eb2d2013-04-23 17:27:38 +09007053 idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE;
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007054 }
7055 sd->last_balance = jiffies;
7056 }
7057 if (need_serialize)
7058 spin_unlock(&balancing);
7059out:
7060 if (time_after(next_balance, sd->last_balance + interval)) {
7061 next_balance = sd->last_balance + interval;
7062 update_next_balance = 1;
7063 }
Jason Lowf48627e2013-09-13 11:26:53 -07007064 }
7065 if (need_decay) {
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007066 /*
Jason Lowf48627e2013-09-13 11:26:53 -07007067 * Ensure the rq-wide value also decays but keep it at a
7068 * reasonable floor to avoid funnies with rq->avg_idle.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007069 */
Jason Lowf48627e2013-09-13 11:26:53 -07007070 rq->max_idle_balance_cost =
7071 max((u64)sysctl_sched_migration_cost, max_cost);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007072 }
Peter Zijlstradce840a2011-04-07 14:09:50 +02007073 rcu_read_unlock();
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007074
7075 /*
7076 * next_balance will be updated only when there is a need.
7077 * When the cpu is attached to null domain for ex, it will not be
7078 * updated.
7079 */
7080 if (likely(update_next_balance))
7081 rq->next_balance = next_balance;
7082}
7083
Frederic Weisbecker3451d022011-08-10 23:21:01 +02007084#ifdef CONFIG_NO_HZ_COMMON
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007085/*
Frederic Weisbecker3451d022011-08-10 23:21:01 +02007086 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007087 * rebalancing for all the cpus for whom scheduler ticks are stopped.
7088 */
Daniel Lezcano208cb162014-01-06 12:34:44 +01007089static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007090{
Daniel Lezcano208cb162014-01-06 12:34:44 +01007091 int this_cpu = this_rq->cpu;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007092 struct rq *rq;
7093 int balance_cpu;
7094
Suresh Siddha1c792db2011-12-01 17:07:32 -08007095 if (idle != CPU_IDLE ||
7096 !test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)))
7097 goto end;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007098
7099 for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
Suresh Siddha8a6d42d2011-12-06 11:19:37 -08007100 if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007101 continue;
7102
7103 /*
7104 * If this cpu gets work to do, stop the load balancing
7105 * work being done for other cpus. Next load
7106 * balancing owner will pick it up.
7107 */
Suresh Siddha1c792db2011-12-01 17:07:32 -08007108 if (need_resched())
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007109 break;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007110
Vincent Guittot5ed4f1d2012-09-13 06:11:26 +02007111 rq = cpu_rq(balance_cpu);
7112
7113 raw_spin_lock_irq(&rq->lock);
7114 update_rq_clock(rq);
7115 update_idle_cpu_load(rq);
7116 raw_spin_unlock_irq(&rq->lock);
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007117
Daniel Lezcanof7ed0a82014-01-06 12:34:43 +01007118 rebalance_domains(rq, CPU_IDLE);
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007119
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007120 if (time_after(this_rq->next_balance, rq->next_balance))
7121 this_rq->next_balance = rq->next_balance;
7122 }
7123 nohz.next_balance = this_rq->next_balance;
Suresh Siddha1c792db2011-12-01 17:07:32 -08007124end:
7125 clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007126}
7127
7128/*
Suresh Siddha0b005cf2011-12-01 17:07:34 -08007129 * Current heuristic for kicking the idle load balancer in the presence
7130 * of an idle cpu is the system.
7131 * - This rq has more than one task.
7132 * - At any scheduler domain level, this cpu's scheduler group has multiple
7133 * busy cpu's exceeding the group's power.
7134 * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler
7135 * domain span are idle.
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007136 */
Daniel Lezcano4a725622014-01-06 12:34:39 +01007137static inline int nohz_kick_needed(struct rq *rq)
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007138{
7139 unsigned long now = jiffies;
Suresh Siddha0b005cf2011-12-01 17:07:34 -08007140 struct sched_domain *sd;
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05307141 struct sched_group_power *sgp;
Daniel Lezcano4a725622014-01-06 12:34:39 +01007142 int nr_busy, cpu = rq->cpu;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007143
Daniel Lezcano4a725622014-01-06 12:34:39 +01007144 if (unlikely(rq->idle_balance))
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007145 return 0;
7146
Suresh Siddha1c792db2011-12-01 17:07:32 -08007147 /*
7148 * We may be recently in ticked or tickless idle mode. At the first
7149 * busy tick after returning from idle, we will update the busy stats.
7150 */
Suresh Siddha69e1e812011-12-01 17:07:33 -08007151 set_cpu_sd_state_busy();
Alex Shic1cc0172012-09-10 15:10:58 +08007152 nohz_balance_exit_idle(cpu);
Suresh Siddha0b005cf2011-12-01 17:07:34 -08007153
7154 /*
7155 * None are in tickless mode and hence no need for NOHZ idle load
7156 * balancing.
7157 */
7158 if (likely(!atomic_read(&nohz.nr_cpus)))
7159 return 0;
Suresh Siddha1c792db2011-12-01 17:07:32 -08007160
7161 if (time_before(now, nohz.next_balance))
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007162 return 0;
7163
Suresh Siddha0b005cf2011-12-01 17:07:34 -08007164 if (rq->nr_running >= 2)
7165 goto need_kick;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007166
Peter Zijlstra067491b2011-12-07 14:32:08 +01007167 rcu_read_lock();
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05307168 sd = rcu_dereference(per_cpu(sd_busy, cpu));
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007169
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05307170 if (sd) {
7171 sgp = sd->groups->sgp;
7172 nr_busy = atomic_read(&sgp->nr_busy_cpus);
7173
7174 if (nr_busy > 1)
Peter Zijlstra067491b2011-12-07 14:32:08 +01007175 goto need_kick_unlock;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007176 }
Preeti U Murthy37dc6b52013-10-30 08:42:52 +05307177
7178 sd = rcu_dereference(per_cpu(sd_asym, cpu));
7179
7180 if (sd && (cpumask_first_and(nohz.idle_cpus_mask,
7181 sched_domain_span(sd)) < cpu))
7182 goto need_kick_unlock;
7183
Peter Zijlstra067491b2011-12-07 14:32:08 +01007184 rcu_read_unlock();
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007185 return 0;
Peter Zijlstra067491b2011-12-07 14:32:08 +01007186
7187need_kick_unlock:
7188 rcu_read_unlock();
Suresh Siddha0b005cf2011-12-01 17:07:34 -08007189need_kick:
7190 return 1;
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007191}
7192#else
Daniel Lezcano208cb162014-01-06 12:34:44 +01007193static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { }
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007194#endif
7195
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007196/*
7197 * run_rebalance_domains is triggered when needed from the scheduler tick.
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007198 * Also triggered for nohz idle balancing (with nohz_balancing_kick set).
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007199 */
7200static void run_rebalance_domains(struct softirq_action *h)
7201{
Daniel Lezcano208cb162014-01-06 12:34:44 +01007202 struct rq *this_rq = this_rq();
Suresh Siddha6eb57e02011-10-03 15:09:01 -07007203 enum cpu_idle_type idle = this_rq->idle_balance ?
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007204 CPU_IDLE : CPU_NOT_IDLE;
7205
Daniel Lezcanof7ed0a82014-01-06 12:34:43 +01007206 rebalance_domains(this_rq, idle);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007207
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007208 /*
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007209 * If this cpu has a pending nohz_balance_kick, then do the
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007210 * balancing on behalf of the other idle cpus whose ticks are
7211 * stopped.
7212 */
Daniel Lezcano208cb162014-01-06 12:34:44 +01007213 nohz_idle_balance(this_rq, idle);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007214}
7215
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007216/*
7217 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007218 */
Daniel Lezcano7caff662014-01-06 12:34:38 +01007219void trigger_load_balance(struct rq *rq)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007220{
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007221 /* Don't need to rebalance while attached to NULL domain */
Daniel Lezcanoc7260992014-01-06 12:34:45 +01007222 if (unlikely(on_null_domain(rq)))
7223 return;
7224
7225 if (time_after_eq(jiffies, rq->next_balance))
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007226 raise_softirq(SCHED_SOFTIRQ);
Frederic Weisbecker3451d022011-08-10 23:21:01 +02007227#ifdef CONFIG_NO_HZ_COMMON
Daniel Lezcanoc7260992014-01-06 12:34:45 +01007228 if (nohz_kick_needed(rq))
Daniel Lezcano0aeeeeb2014-01-06 12:34:42 +01007229 nohz_balancer_kick();
Venkatesh Pallipadi83cd4fe2010-05-21 17:09:41 -07007230#endif
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01007231}
7232
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01007233static void rq_online_fair(struct rq *rq)
7234{
7235 update_sysctl();
7236}
7237
7238static void rq_offline_fair(struct rq *rq)
7239{
7240 update_sysctl();
Peter Boonstoppela4c96ae2012-08-09 15:34:47 -07007241
7242 /* Ensure any throttled groups are reachable by pick_next_task */
7243 unthrottle_offline_cfs_rqs(rq);
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01007244}
7245
Dhaval Giani55e12e52008-06-24 23:39:43 +05307246#endif /* CONFIG_SMP */
Peter Williamse1d14842007-10-24 18:23:51 +02007247
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007248/*
7249 * scheduler tick hitting a task of our scheduling class:
7250 */
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01007251static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007252{
7253 struct cfs_rq *cfs_rq;
7254 struct sched_entity *se = &curr->se;
7255
7256 for_each_sched_entity(se) {
7257 cfs_rq = cfs_rq_of(se);
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01007258 entity_tick(cfs_rq, se, queued);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007259 }
Ben Segall18bf2802012-10-04 12:51:20 +02007260
Dave Kleikamp10e84b92013-07-31 13:53:35 -07007261 if (numabalancing_enabled)
Peter Zijlstracbee9f82012-10-25 14:16:43 +02007262 task_tick_numa(rq, curr);
Linus Torvalds3d59eeb2012-12-16 14:33:25 -08007263
Ben Segall18bf2802012-10-04 12:51:20 +02007264 update_rq_runnable_avg(rq, 1);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007265}
7266
7267/*
Peter Zijlstracd29fe62009-11-27 17:32:46 +01007268 * called on fork with the child task as argument from the parent's context
7269 * - child not yet on the tasklist
7270 * - preemption disabled
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007271 */
Peter Zijlstracd29fe62009-11-27 17:32:46 +01007272static void task_fork_fair(struct task_struct *p)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007273{
Daisuke Nishimura4fc420c2011-12-15 14:36:55 +09007274 struct cfs_rq *cfs_rq;
7275 struct sched_entity *se = &p->se, *curr;
Ingo Molnar00bf7bf2007-10-15 17:00:14 +02007276 int this_cpu = smp_processor_id();
Peter Zijlstracd29fe62009-11-27 17:32:46 +01007277 struct rq *rq = this_rq();
7278 unsigned long flags;
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007279
Thomas Gleixner05fa7852009-11-17 14:28:38 +01007280 raw_spin_lock_irqsave(&rq->lock, flags);
Peter Zijlstracd29fe62009-11-27 17:32:46 +01007281
Peter Zijlstra861d0342010-08-19 13:31:43 +02007282 update_rq_clock(rq);
7283
Daisuke Nishimura4fc420c2011-12-15 14:36:55 +09007284 cfs_rq = task_cfs_rq(current);
7285 curr = cfs_rq->curr;
7286
Daisuke Nishimura6c9a27f2013-09-10 18:16:36 +09007287 /*
7288 * Not only the cpu but also the task_group of the parent might have
7289 * been changed after parent->se.parent,cfs_rq were copied to
7290 * child->se.parent,cfs_rq. So call __set_task_cpu() to make those
7291 * of child point to valid ones.
7292 */
7293 rcu_read_lock();
7294 __set_task_cpu(p, this_cpu);
7295 rcu_read_unlock();
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007296
Ting Yang7109c4422007-08-28 12:53:24 +02007297 update_curr(cfs_rq);
Peter Zijlstracd29fe62009-11-27 17:32:46 +01007298
Mike Galbraithb5d9d732009-09-08 11:12:28 +02007299 if (curr)
7300 se->vruntime = curr->vruntime;
Peter Zijlstraaeb73b02007-10-15 17:00:05 +02007301 place_entity(cfs_rq, se, 1);
Peter Zijlstra4d78e7b2007-10-15 17:00:04 +02007302
Peter Zijlstracd29fe62009-11-27 17:32:46 +01007303 if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) {
Dmitry Adamushko87fefa32007-10-15 17:00:08 +02007304 /*
Ingo Molnaredcb60a2007-10-15 17:00:08 +02007305 * Upon rescheduling, sched_class::put_prev_task() will place
7306 * 'current' within the tree based on its new key value.
7307 */
Peter Zijlstra4d78e7b2007-10-15 17:00:04 +02007308 swap(curr->vruntime, se->vruntime);
Bharata B Raoaec0a512008-08-28 14:42:49 +05307309 resched_task(rq->curr);
Peter Zijlstra4d78e7b2007-10-15 17:00:04 +02007310 }
7311
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01007312 se->vruntime -= cfs_rq->min_vruntime;
7313
Thomas Gleixner05fa7852009-11-17 14:28:38 +01007314 raw_spin_unlock_irqrestore(&rq->lock, flags);
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007315}
7316
Steven Rostedtcb469842008-01-25 21:08:22 +01007317/*
7318 * Priority of the task has changed. Check to see if we preempt
7319 * the current task.
7320 */
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007321static void
7322prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio)
Steven Rostedtcb469842008-01-25 21:08:22 +01007323{
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007324 if (!p->se.on_rq)
7325 return;
7326
Steven Rostedtcb469842008-01-25 21:08:22 +01007327 /*
7328 * Reschedule if we are currently running on this runqueue and
7329 * our priority decreased, or if we are not currently running on
7330 * this runqueue and our priority is higher than the current's
7331 */
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007332 if (rq->curr == p) {
Steven Rostedtcb469842008-01-25 21:08:22 +01007333 if (p->prio > oldprio)
7334 resched_task(rq->curr);
7335 } else
Peter Zijlstra15afe092008-09-20 23:38:02 +02007336 check_preempt_curr(rq, p, 0);
Steven Rostedtcb469842008-01-25 21:08:22 +01007337}
7338
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007339static void switched_from_fair(struct rq *rq, struct task_struct *p)
7340{
7341 struct sched_entity *se = &p->se;
7342 struct cfs_rq *cfs_rq = cfs_rq_of(se);
7343
7344 /*
George McCollister791c9e02014-02-18 17:56:51 -06007345 * Ensure the task's vruntime is normalized, so that when it's
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007346 * switched back to the fair class the enqueue_entity(.flags=0) will
7347 * do the right thing.
7348 *
George McCollister791c9e02014-02-18 17:56:51 -06007349 * If it's on_rq, then the dequeue_entity(.flags=0) will already
7350 * have normalized the vruntime, if it's !on_rq, then only when
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007351 * the task is sleeping will it still have non-normalized vruntime.
7352 */
George McCollister791c9e02014-02-18 17:56:51 -06007353 if (!p->on_rq && p->state != TASK_RUNNING) {
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007354 /*
7355 * Fix up our vruntime so that the current sleep doesn't
7356 * cause 'unlimited' sleep bonus.
7357 */
7358 place_entity(cfs_rq, se, 0);
7359 se->vruntime -= cfs_rq->min_vruntime;
7360 }
Paul Turner9ee474f2012-10-04 13:18:30 +02007361
Alex Shi141965c2013-06-26 13:05:39 +08007362#ifdef CONFIG_SMP
Paul Turner9ee474f2012-10-04 13:18:30 +02007363 /*
7364 * Remove our load from contribution when we leave sched_fair
7365 * and ensure we don't carry in an old decay_count if we
7366 * switch back.
7367 */
Kirill Tkhai87e3c8a2013-07-21 04:32:07 +04007368 if (se->avg.decay_count) {
7369 __synchronize_entity_decay(se);
7370 subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib);
Paul Turner9ee474f2012-10-04 13:18:30 +02007371 }
7372#endif
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007373}
7374
Steven Rostedtcb469842008-01-25 21:08:22 +01007375/*
7376 * We switched to the sched_fair class.
7377 */
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007378static void switched_to_fair(struct rq *rq, struct task_struct *p)
Steven Rostedtcb469842008-01-25 21:08:22 +01007379{
Michael wangeb7a59b2014-02-20 11:14:53 +08007380 struct sched_entity *se = &p->se;
7381#ifdef CONFIG_FAIR_GROUP_SCHED
7382 /*
7383 * Since the real-depth could have been changed (only FAIR
7384 * class maintain depth value), reset depth properly.
7385 */
7386 se->depth = se->parent ? se->parent->depth + 1 : 0;
7387#endif
7388 if (!se->on_rq)
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007389 return;
7390
Steven Rostedtcb469842008-01-25 21:08:22 +01007391 /*
7392 * We were most likely switched from sched_rt, so
7393 * kick off the schedule if running, otherwise just see
7394 * if we can still preempt the current task.
7395 */
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007396 if (rq->curr == p)
Steven Rostedtcb469842008-01-25 21:08:22 +01007397 resched_task(rq->curr);
7398 else
Peter Zijlstra15afe092008-09-20 23:38:02 +02007399 check_preempt_curr(rq, p, 0);
Steven Rostedtcb469842008-01-25 21:08:22 +01007400}
7401
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02007402/* Account for a task changing its policy or group.
7403 *
7404 * This routine is mostly called to set cfs_rq->curr field when a task
7405 * migrates between groups/classes.
7406 */
7407static void set_curr_task_fair(struct rq *rq)
7408{
7409 struct sched_entity *se = &rq->curr->se;
7410
Paul Turnerec12cb72011-07-21 09:43:30 -07007411 for_each_sched_entity(se) {
7412 struct cfs_rq *cfs_rq = cfs_rq_of(se);
7413
7414 set_next_entity(cfs_rq, se);
7415 /* ensure bandwidth has been allocated on our new cfs_rq */
7416 account_cfs_rq_runtime(cfs_rq, 0);
7417 }
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02007418}
7419
Peter Zijlstra029632f2011-10-25 10:00:11 +02007420void init_cfs_rq(struct cfs_rq *cfs_rq)
7421{
7422 cfs_rq->tasks_timeline = RB_ROOT;
Peter Zijlstra029632f2011-10-25 10:00:11 +02007423 cfs_rq->min_vruntime = (u64)(-(1LL << 20));
7424#ifndef CONFIG_64BIT
7425 cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
7426#endif
Alex Shi141965c2013-06-26 13:05:39 +08007427#ifdef CONFIG_SMP
Paul Turner9ee474f2012-10-04 13:18:30 +02007428 atomic64_set(&cfs_rq->decay_counter, 1);
Alex Shi25099402013-06-20 10:18:55 +08007429 atomic_long_set(&cfs_rq->removed_load, 0);
Paul Turner9ee474f2012-10-04 13:18:30 +02007430#endif
Peter Zijlstra029632f2011-10-25 10:00:11 +02007431}
7432
Peter Zijlstra810b3812008-02-29 15:21:01 -05007433#ifdef CONFIG_FAIR_GROUP_SCHED
Peter Zijlstrab2b5ce02010-10-15 15:24:15 +02007434static void task_move_group_fair(struct task_struct *p, int on_rq)
Peter Zijlstra810b3812008-02-29 15:21:01 -05007435{
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007436 struct sched_entity *se = &p->se;
Paul Turneraff3e492012-10-04 13:18:30 +02007437 struct cfs_rq *cfs_rq;
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007438
Peter Zijlstrab2b5ce02010-10-15 15:24:15 +02007439 /*
7440 * If the task was not on the rq at the time of this cgroup movement
7441 * it must have been asleep, sleeping tasks keep their ->vruntime
7442 * absolute on their old rq until wakeup (needed for the fair sleeper
7443 * bonus in place_entity()).
7444 *
7445 * If it was on the rq, we've just 'preempted' it, which does convert
7446 * ->vruntime to a relative base.
7447 *
7448 * Make sure both cases convert their relative position when migrating
7449 * to another cgroup's rq. This does somewhat interfere with the
7450 * fair sleeper stuff for the first placement, but who cares.
7451 */
Daisuke Nishimura7ceff012011-12-15 14:36:07 +09007452 /*
7453 * When !on_rq, vruntime of the task has usually NOT been normalized.
7454 * But there are some cases where it has already been normalized:
7455 *
7456 * - Moving a forked child which is waiting for being woken up by
7457 * wake_up_new_task().
Daisuke Nishimura62af3782011-12-15 14:37:41 +09007458 * - Moving a task which has been woken up by try_to_wake_up() and
7459 * waiting for actually being woken up by sched_ttwu_pending().
Daisuke Nishimura7ceff012011-12-15 14:36:07 +09007460 *
7461 * To prevent boost or penalty in the new cfs_rq caused by delta
7462 * min_vruntime between the two cfs_rqs, we skip vruntime adjustment.
7463 */
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007464 if (!on_rq && (!se->sum_exec_runtime || p->state == TASK_WAKING))
Daisuke Nishimura7ceff012011-12-15 14:36:07 +09007465 on_rq = 1;
7466
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01007467 if (!on_rq)
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007468 se->vruntime -= cfs_rq_of(se)->min_vruntime;
Peter Zijlstrab2b5ce02010-10-15 15:24:15 +02007469 set_task_rq(p, task_cpu(p));
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007470 se->depth = se->parent ? se->parent->depth + 1 : 0;
Paul Turneraff3e492012-10-04 13:18:30 +02007471 if (!on_rq) {
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007472 cfs_rq = cfs_rq_of(se);
7473 se->vruntime += cfs_rq->min_vruntime;
Paul Turneraff3e492012-10-04 13:18:30 +02007474#ifdef CONFIG_SMP
7475 /*
7476 * migrate_task_rq_fair() will have removed our previous
7477 * contribution, but we must synchronize for ongoing future
7478 * decay.
7479 */
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007480 se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter);
7481 cfs_rq->blocked_load_avg += se->avg.load_avg_contrib;
Paul Turneraff3e492012-10-04 13:18:30 +02007482#endif
7483 }
Peter Zijlstra810b3812008-02-29 15:21:01 -05007484}
Peter Zijlstra029632f2011-10-25 10:00:11 +02007485
7486void free_fair_sched_group(struct task_group *tg)
7487{
7488 int i;
7489
7490 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
7491
7492 for_each_possible_cpu(i) {
7493 if (tg->cfs_rq)
7494 kfree(tg->cfs_rq[i]);
7495 if (tg->se)
7496 kfree(tg->se[i]);
7497 }
7498
7499 kfree(tg->cfs_rq);
7500 kfree(tg->se);
7501}
7502
7503int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
7504{
7505 struct cfs_rq *cfs_rq;
7506 struct sched_entity *se;
7507 int i;
7508
7509 tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
7510 if (!tg->cfs_rq)
7511 goto err;
7512 tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
7513 if (!tg->se)
7514 goto err;
7515
7516 tg->shares = NICE_0_LOAD;
7517
7518 init_cfs_bandwidth(tg_cfs_bandwidth(tg));
7519
7520 for_each_possible_cpu(i) {
7521 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
7522 GFP_KERNEL, cpu_to_node(i));
7523 if (!cfs_rq)
7524 goto err;
7525
7526 se = kzalloc_node(sizeof(struct sched_entity),
7527 GFP_KERNEL, cpu_to_node(i));
7528 if (!se)
7529 goto err_free_rq;
7530
7531 init_cfs_rq(cfs_rq);
7532 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
7533 }
7534
7535 return 1;
7536
7537err_free_rq:
7538 kfree(cfs_rq);
7539err:
7540 return 0;
7541}
7542
7543void unregister_fair_sched_group(struct task_group *tg, int cpu)
7544{
7545 struct rq *rq = cpu_rq(cpu);
7546 unsigned long flags;
7547
7548 /*
7549 * Only empty task groups can be destroyed; so we can speculatively
7550 * check on_list without danger of it being re-added.
7551 */
7552 if (!tg->cfs_rq[cpu]->on_list)
7553 return;
7554
7555 raw_spin_lock_irqsave(&rq->lock, flags);
7556 list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
7557 raw_spin_unlock_irqrestore(&rq->lock, flags);
7558}
7559
7560void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
7561 struct sched_entity *se, int cpu,
7562 struct sched_entity *parent)
7563{
7564 struct rq *rq = cpu_rq(cpu);
7565
7566 cfs_rq->tg = tg;
7567 cfs_rq->rq = rq;
Peter Zijlstra029632f2011-10-25 10:00:11 +02007568 init_cfs_rq_runtime(cfs_rq);
7569
7570 tg->cfs_rq[cpu] = cfs_rq;
7571 tg->se[cpu] = se;
7572
7573 /* se could be NULL for root_task_group */
7574 if (!se)
7575 return;
7576
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007577 if (!parent) {
Peter Zijlstra029632f2011-10-25 10:00:11 +02007578 se->cfs_rq = &rq->cfs;
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007579 se->depth = 0;
7580 } else {
Peter Zijlstra029632f2011-10-25 10:00:11 +02007581 se->cfs_rq = parent->my_q;
Peter Zijlstrafed14d42012-02-11 06:05:00 +01007582 se->depth = parent->depth + 1;
7583 }
Peter Zijlstra029632f2011-10-25 10:00:11 +02007584
7585 se->my_q = cfs_rq;
Paul Turner0ac9b1c2013-10-16 11:16:27 -07007586 /* guarantee group entities always have weight */
7587 update_load_set(&se->load, NICE_0_LOAD);
Peter Zijlstra029632f2011-10-25 10:00:11 +02007588 se->parent = parent;
7589}
7590
7591static DEFINE_MUTEX(shares_mutex);
7592
7593int sched_group_set_shares(struct task_group *tg, unsigned long shares)
7594{
7595 int i;
7596 unsigned long flags;
7597
7598 /*
7599 * We can't change the weight of the root cgroup.
7600 */
7601 if (!tg->se[0])
7602 return -EINVAL;
7603
7604 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
7605
7606 mutex_lock(&shares_mutex);
7607 if (tg->shares == shares)
7608 goto done;
7609
7610 tg->shares = shares;
7611 for_each_possible_cpu(i) {
7612 struct rq *rq = cpu_rq(i);
7613 struct sched_entity *se;
7614
7615 se = tg->se[i];
7616 /* Propagate contribution to hierarchy */
7617 raw_spin_lock_irqsave(&rq->lock, flags);
Frederic Weisbecker71b1da42013-04-12 01:50:59 +02007618
7619 /* Possible calls to update_curr() need rq clock */
7620 update_rq_clock(rq);
Linus Torvalds17bc14b2012-12-14 07:20:43 -08007621 for_each_sched_entity(se)
Peter Zijlstra029632f2011-10-25 10:00:11 +02007622 update_cfs_shares(group_cfs_rq(se));
7623 raw_spin_unlock_irqrestore(&rq->lock, flags);
7624 }
7625
7626done:
7627 mutex_unlock(&shares_mutex);
7628 return 0;
7629}
7630#else /* CONFIG_FAIR_GROUP_SCHED */
7631
7632void free_fair_sched_group(struct task_group *tg) { }
7633
7634int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
7635{
7636 return 1;
7637}
7638
7639void unregister_fair_sched_group(struct task_group *tg, int cpu) { }
7640
7641#endif /* CONFIG_FAIR_GROUP_SCHED */
7642
Peter Zijlstra810b3812008-02-29 15:21:01 -05007643
H Hartley Sweeten6d686f42010-01-13 20:21:52 -07007644static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
Peter Williams0d721ce2009-09-21 01:31:53 +00007645{
7646 struct sched_entity *se = &task->se;
Peter Williams0d721ce2009-09-21 01:31:53 +00007647 unsigned int rr_interval = 0;
7648
7649 /*
7650 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
7651 * idle runqueue:
7652 */
Peter Williams0d721ce2009-09-21 01:31:53 +00007653 if (rq->cfs.load.weight)
Zhu Yanhaia59f4e02013-01-08 12:56:52 +08007654 rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se));
Peter Williams0d721ce2009-09-21 01:31:53 +00007655
7656 return rr_interval;
7657}
7658
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007659/*
7660 * All the scheduling class methods:
7661 */
Peter Zijlstra029632f2011-10-25 10:00:11 +02007662const struct sched_class fair_sched_class = {
Ingo Molnar5522d5d2007-10-15 17:00:12 +02007663 .next = &idle_sched_class,
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007664 .enqueue_task = enqueue_task_fair,
7665 .dequeue_task = dequeue_task_fair,
7666 .yield_task = yield_task_fair,
Mike Galbraithd95f4122011-02-01 09:50:51 -05007667 .yield_to_task = yield_to_task_fair,
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007668
Ingo Molnar2e09bf52007-10-15 17:00:05 +02007669 .check_preempt_curr = check_preempt_wakeup,
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007670
7671 .pick_next_task = pick_next_task_fair,
7672 .put_prev_task = put_prev_task_fair,
7673
Peter Williams681f3e62007-10-24 18:23:51 +02007674#ifdef CONFIG_SMP
Li Zefan4ce72a22008-10-22 15:25:26 +08007675 .select_task_rq = select_task_rq_fair,
Paul Turner0a74bef2012-10-04 13:18:30 +02007676 .migrate_task_rq = migrate_task_rq_fair,
Alex Shi141965c2013-06-26 13:05:39 +08007677
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01007678 .rq_online = rq_online_fair,
7679 .rq_offline = rq_offline_fair,
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01007680
7681 .task_waking = task_waking_fair,
Peter Williams681f3e62007-10-24 18:23:51 +02007682#endif
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007683
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02007684 .set_curr_task = set_curr_task_fair,
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007685 .task_tick = task_tick_fair,
Peter Zijlstracd29fe62009-11-27 17:32:46 +01007686 .task_fork = task_fork_fair,
Steven Rostedtcb469842008-01-25 21:08:22 +01007687
7688 .prio_changed = prio_changed_fair,
Peter Zijlstrada7a7352011-01-17 17:03:27 +01007689 .switched_from = switched_from_fair,
Steven Rostedtcb469842008-01-25 21:08:22 +01007690 .switched_to = switched_to_fair,
Peter Zijlstra810b3812008-02-29 15:21:01 -05007691
Peter Williams0d721ce2009-09-21 01:31:53 +00007692 .get_rr_interval = get_rr_interval_fair,
7693
Peter Zijlstra810b3812008-02-29 15:21:01 -05007694#ifdef CONFIG_FAIR_GROUP_SCHED
Peter Zijlstrab2b5ce02010-10-15 15:24:15 +02007695 .task_move_group = task_move_group_fair,
Peter Zijlstra810b3812008-02-29 15:21:01 -05007696#endif
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007697};
7698
7699#ifdef CONFIG_SCHED_DEBUG
Peter Zijlstra029632f2011-10-25 10:00:11 +02007700void print_cfs_stats(struct seq_file *m, int cpu)
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007701{
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007702 struct cfs_rq *cfs_rq;
7703
Peter Zijlstra5973e5b2008-01-25 21:08:34 +01007704 rcu_read_lock();
Ingo Molnarc3b64f12007-08-09 11:16:51 +02007705 for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq)
Ingo Molnar5cef9ec2007-08-09 11:16:47 +02007706 print_cfs_rq(m, cpu, cfs_rq);
Peter Zijlstra5973e5b2008-01-25 21:08:34 +01007707 rcu_read_unlock();
Ingo Molnarbf0f6f22007-07-09 18:51:58 +02007708}
7709#endif
Peter Zijlstra029632f2011-10-25 10:00:11 +02007710
7711__init void init_sched_fair_class(void)
7712{
7713#ifdef CONFIG_SMP
7714 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
7715
Frederic Weisbecker3451d022011-08-10 23:21:01 +02007716#ifdef CONFIG_NO_HZ_COMMON
Diwakar Tundlam554ceca2012-03-07 14:44:26 -08007717 nohz.next_balance = jiffies;
Peter Zijlstra029632f2011-10-25 10:00:11 +02007718 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
Suresh Siddha71325962012-01-19 18:28:57 -08007719 cpu_notifier(sched_ilb_notifier, 0);
Peter Zijlstra029632f2011-10-25 10:00:11 +02007720#endif
7721#endif /* SMP */
7722
7723}