Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) |
| 3 | * |
| 4 | * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
| 5 | * |
| 6 | * Interactivity improvements by Mike Galbraith |
| 7 | * (C) 2007 Mike Galbraith <efault@gmx.de> |
| 8 | * |
| 9 | * Various enhancements by Dmitry Adamushko. |
| 10 | * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> |
| 11 | * |
| 12 | * Group scheduling enhancements by Srivatsa Vaddagiri |
| 13 | * Copyright IBM Corporation, 2007 |
| 14 | * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> |
| 15 | * |
| 16 | * Scaled math optimizations by Thomas Gleixner |
| 17 | * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 18 | * |
| 19 | * Adaptive scheduling granularity, math enhancements by Peter Zijlstra |
Peter Zijlstra | 90eec10 | 2015-11-16 11:08:45 +0100 | [diff] [blame] | 20 | * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 21 | */ |
| 22 | |
Ingo Molnar | 589ee62 | 2017-02-04 00:16:44 +0100 | [diff] [blame] | 23 | #include <linux/sched/mm.h> |
Ingo Molnar | 105ab3d | 2017-02-01 16:36:40 +0100 | [diff] [blame] | 24 | #include <linux/sched/topology.h> |
| 25 | |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 26 | #include <linux/latencytop.h> |
Sisir Koppaka | 3436ae1 | 2011-03-26 18:22:55 +0530 | [diff] [blame] | 27 | #include <linux/cpumask.h> |
Nicolas Pitre | 83a0a96 | 2014-09-04 11:32:10 -0400 | [diff] [blame] | 28 | #include <linux/cpuidle.h> |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 29 | #include <linux/slab.h> |
| 30 | #include <linux/profile.h> |
| 31 | #include <linux/interrupt.h> |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 32 | #include <linux/mempolicy.h> |
Mel Gorman | e14808b | 2012-11-19 10:59:15 +0000 | [diff] [blame] | 33 | #include <linux/migrate.h> |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 34 | #include <linux/task_work.h> |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 35 | |
| 36 | #include <trace/events/sched.h> |
| 37 | |
| 38 | #include "sched.h" |
Arjan van de Ven | 9745512 | 2008-01-25 21:08:34 +0100 | [diff] [blame] | 39 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 40 | /* |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 41 | * Targeted preemption latency for CPU-bound tasks: |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 42 | * |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 43 | * NOTE: this latency value is not the same as the concept of |
Ingo Molnar | d274a4c | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 44 | * 'timeslice length' - timeslices in CFS are of variable length |
| 45 | * and have no persistent notion like in traditional, time-slice |
| 46 | * based scheduling concepts. |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 47 | * |
Ingo Molnar | d274a4c | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 48 | * (to see the precise effective timeslice length of your workload, |
| 49 | * run vmstat and monitor the context-switches (cs) field) |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 50 | * |
| 51 | * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 52 | */ |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 53 | unsigned int sysctl_sched_latency = 6000000ULL; |
| 54 | unsigned int normalized_sysctl_sched_latency = 6000000ULL; |
Ingo Molnar | 2bd8e6d | 2007-10-15 17:00:02 +0200 | [diff] [blame] | 55 | |
| 56 | /* |
Christian Ehrhardt | 1983a92 | 2009-11-30 12:16:47 +0100 | [diff] [blame] | 57 | * The initial- and re-scaling of tunables is configurable |
Christian Ehrhardt | 1983a92 | 2009-11-30 12:16:47 +0100 | [diff] [blame] | 58 | * |
| 59 | * Options are: |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 60 | * |
| 61 | * SCHED_TUNABLESCALING_NONE - unscaled, always *1 |
| 62 | * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) |
| 63 | * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus |
| 64 | * |
| 65 | * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) |
Christian Ehrhardt | 1983a92 | 2009-11-30 12:16:47 +0100 | [diff] [blame] | 66 | */ |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 67 | enum sched_tunable_scaling sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; |
Christian Ehrhardt | 1983a92 | 2009-11-30 12:16:47 +0100 | [diff] [blame] | 68 | |
| 69 | /* |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 70 | * Minimal preemption granularity for CPU-bound tasks: |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 71 | * |
Takuya Yoshikawa | 864616e | 2010-10-14 16:09:13 +0900 | [diff] [blame] | 72 | * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 73 | */ |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 74 | unsigned int sysctl_sched_min_granularity = 750000ULL; |
| 75 | unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 76 | |
| 77 | /* |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 78 | * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 79 | */ |
Ingo Molnar | 0bf377b | 2010-09-12 08:14:52 +0200 | [diff] [blame] | 80 | static unsigned int sched_nr_latency = 8; |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 81 | |
| 82 | /* |
Mike Galbraith | 2bba22c | 2009-09-09 15:41:37 +0200 | [diff] [blame] | 83 | * After fork, child runs first. If set to 0 (default) then |
Ingo Molnar | 2bd8e6d | 2007-10-15 17:00:02 +0200 | [diff] [blame] | 84 | * parent will (try to) run first. |
| 85 | */ |
Mike Galbraith | 2bba22c | 2009-09-09 15:41:37 +0200 | [diff] [blame] | 86 | unsigned int sysctl_sched_child_runs_first __read_mostly; |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 87 | |
| 88 | /* |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 89 | * SCHED_OTHER wake-up granularity. |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 90 | * |
| 91 | * This option delays the preemption effects of decoupled workloads |
| 92 | * and reduces their over-scheduling. Synchronous workloads will still |
| 93 | * have immediate wakeup/sleep latencies. |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 94 | * |
| 95 | * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 96 | */ |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 97 | unsigned int sysctl_sched_wakeup_granularity = 1000000UL; |
| 98 | unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 99 | |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 100 | const_debug unsigned int sysctl_sched_migration_cost = 500000UL; |
Ingo Molnar | da84d96 | 2007-10-15 17:00:18 +0200 | [diff] [blame] | 101 | |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 102 | #ifdef CONFIG_SMP |
| 103 | /* |
| 104 | * For asym packing, by default the lower numbered cpu has higher priority. |
| 105 | */ |
| 106 | int __weak arch_asym_cpu_priority(int cpu) |
| 107 | { |
| 108 | return -cpu; |
| 109 | } |
| 110 | #endif |
| 111 | |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 112 | #ifdef CONFIG_CFS_BANDWIDTH |
| 113 | /* |
| 114 | * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool |
| 115 | * each time a cfs_rq requests quota. |
| 116 | * |
| 117 | * Note: in the case that the slice exceeds the runtime remaining (either due |
| 118 | * to consumption or the quota being specified to be smaller than the slice) |
| 119 | * we will always only issue the remaining available time. |
| 120 | * |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 121 | * (default: 5 msec, units: microseconds) |
| 122 | */ |
| 123 | unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 124 | #endif |
| 125 | |
Morten Rasmussen | 3273163 | 2016-07-25 14:34:26 +0100 | [diff] [blame] | 126 | /* |
| 127 | * The margin used when comparing utilization with CPU capacity: |
Morten Rasmussen | 893c5d2 | 2016-10-14 14:41:12 +0100 | [diff] [blame] | 128 | * util * margin < capacity * 1024 |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 129 | * |
| 130 | * (default: ~20%) |
Morten Rasmussen | 3273163 | 2016-07-25 14:34:26 +0100 | [diff] [blame] | 131 | */ |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 132 | unsigned int capacity_margin = 1280; |
Morten Rasmussen | 3273163 | 2016-07-25 14:34:26 +0100 | [diff] [blame] | 133 | |
Paul Gortmaker | 8527632 | 2013-04-19 15:10:50 -0400 | [diff] [blame] | 134 | static inline void update_load_add(struct load_weight *lw, unsigned long inc) |
| 135 | { |
| 136 | lw->weight += inc; |
| 137 | lw->inv_weight = 0; |
| 138 | } |
| 139 | |
| 140 | static inline void update_load_sub(struct load_weight *lw, unsigned long dec) |
| 141 | { |
| 142 | lw->weight -= dec; |
| 143 | lw->inv_weight = 0; |
| 144 | } |
| 145 | |
| 146 | static inline void update_load_set(struct load_weight *lw, unsigned long w) |
| 147 | { |
| 148 | lw->weight = w; |
| 149 | lw->inv_weight = 0; |
| 150 | } |
| 151 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 152 | /* |
| 153 | * Increase the granularity value when there are more CPUs, |
| 154 | * because with more CPUs the 'effective latency' as visible |
| 155 | * to users decreases. But the relationship is not linear, |
| 156 | * so pick a second-best guess by going with the log2 of the |
| 157 | * number of CPUs. |
| 158 | * |
| 159 | * This idea comes from the SD scheduler of Con Kolivas: |
| 160 | */ |
Nicholas Mc Guire | 58ac93e | 2015-05-15 21:05:42 +0200 | [diff] [blame] | 161 | static unsigned int get_update_sysctl_factor(void) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 162 | { |
Nicholas Mc Guire | 58ac93e | 2015-05-15 21:05:42 +0200 | [diff] [blame] | 163 | unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 164 | unsigned int factor; |
| 165 | |
| 166 | switch (sysctl_sched_tunable_scaling) { |
| 167 | case SCHED_TUNABLESCALING_NONE: |
| 168 | factor = 1; |
| 169 | break; |
| 170 | case SCHED_TUNABLESCALING_LINEAR: |
| 171 | factor = cpus; |
| 172 | break; |
| 173 | case SCHED_TUNABLESCALING_LOG: |
| 174 | default: |
| 175 | factor = 1 + ilog2(cpus); |
| 176 | break; |
| 177 | } |
| 178 | |
| 179 | return factor; |
| 180 | } |
| 181 | |
| 182 | static void update_sysctl(void) |
| 183 | { |
| 184 | unsigned int factor = get_update_sysctl_factor(); |
| 185 | |
| 186 | #define SET_SYSCTL(name) \ |
| 187 | (sysctl_##name = (factor) * normalized_sysctl_##name) |
| 188 | SET_SYSCTL(sched_min_granularity); |
| 189 | SET_SYSCTL(sched_latency); |
| 190 | SET_SYSCTL(sched_wakeup_granularity); |
| 191 | #undef SET_SYSCTL |
| 192 | } |
| 193 | |
| 194 | void sched_init_granularity(void) |
| 195 | { |
| 196 | update_sysctl(); |
| 197 | } |
| 198 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 199 | #define WMULT_CONST (~0U) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 200 | #define WMULT_SHIFT 32 |
| 201 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 202 | static void __update_inv_weight(struct load_weight *lw) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 203 | { |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 204 | unsigned long w; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 205 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 206 | if (likely(lw->inv_weight)) |
| 207 | return; |
| 208 | |
| 209 | w = scale_load_down(lw->weight); |
| 210 | |
| 211 | if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) |
| 212 | lw->inv_weight = 1; |
| 213 | else if (unlikely(!w)) |
| 214 | lw->inv_weight = WMULT_CONST; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 215 | else |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 216 | lw->inv_weight = WMULT_CONST / w; |
| 217 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 218 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 219 | /* |
| 220 | * delta_exec * weight / lw.weight |
| 221 | * OR |
| 222 | * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT |
| 223 | * |
Yuyang Du | 1c3de5e | 2016-03-30 07:07:51 +0800 | [diff] [blame] | 224 | * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 225 | * we're guaranteed shift stays positive because inv_weight is guaranteed to |
| 226 | * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. |
| 227 | * |
| 228 | * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus |
| 229 | * weight/lw.weight <= 1, and therefore our shift will also be positive. |
| 230 | */ |
| 231 | static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) |
| 232 | { |
| 233 | u64 fact = scale_load_down(weight); |
| 234 | int shift = WMULT_SHIFT; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 235 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 236 | __update_inv_weight(lw); |
| 237 | |
| 238 | if (unlikely(fact >> 32)) { |
| 239 | while (fact >> 32) { |
| 240 | fact >>= 1; |
| 241 | shift--; |
| 242 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 243 | } |
| 244 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 245 | /* hint to use a 32x32->64 mul */ |
| 246 | fact = (u64)(u32)fact * lw->inv_weight; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 247 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 248 | while (fact >> 32) { |
| 249 | fact >>= 1; |
| 250 | shift--; |
| 251 | } |
| 252 | |
| 253 | return mul_u64_u32_shr(delta_exec, fact, shift); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 254 | } |
| 255 | |
| 256 | |
| 257 | const struct sched_class fair_sched_class; |
Peter Zijlstra | a4c2f00 | 2008-10-17 19:27:03 +0200 | [diff] [blame] | 258 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 259 | /************************************************************** |
| 260 | * CFS operations on generic schedulable entities: |
| 261 | */ |
| 262 | |
| 263 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 264 | |
| 265 | /* cpu runqueue to which this cfs_rq is attached */ |
| 266 | static inline struct rq *rq_of(struct cfs_rq *cfs_rq) |
| 267 | { |
| 268 | return cfs_rq->rq; |
| 269 | } |
| 270 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 271 | /* An entity is a task if it doesn't "own" a runqueue */ |
| 272 | #define entity_is_task(se) (!se->my_q) |
| 273 | |
Peter Zijlstra | 8f48894 | 2009-07-24 12:25:30 +0200 | [diff] [blame] | 274 | static inline struct task_struct *task_of(struct sched_entity *se) |
| 275 | { |
Peter Zijlstra | 9148a3a | 2016-09-20 22:34:51 +0200 | [diff] [blame] | 276 | SCHED_WARN_ON(!entity_is_task(se)); |
Peter Zijlstra | 8f48894 | 2009-07-24 12:25:30 +0200 | [diff] [blame] | 277 | return container_of(se, struct task_struct, se); |
| 278 | } |
| 279 | |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 280 | /* Walk up scheduling entities hierarchy */ |
| 281 | #define for_each_sched_entity(se) \ |
| 282 | for (; se; se = se->parent) |
| 283 | |
| 284 | static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) |
| 285 | { |
| 286 | return p->se.cfs_rq; |
| 287 | } |
| 288 | |
| 289 | /* runqueue on which this entity is (to be) queued */ |
| 290 | static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) |
| 291 | { |
| 292 | return se->cfs_rq; |
| 293 | } |
| 294 | |
| 295 | /* runqueue "owned" by this group */ |
| 296 | static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) |
| 297 | { |
| 298 | return grp->my_q; |
| 299 | } |
| 300 | |
Peter Zijlstra | 3d4b47b | 2010-11-15 15:47:01 -0800 | [diff] [blame] | 301 | static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) |
| 302 | { |
| 303 | if (!cfs_rq->on_list) { |
Vincent Guittot | 9c2791f | 2016-11-08 10:53:43 +0100 | [diff] [blame] | 304 | struct rq *rq = rq_of(cfs_rq); |
| 305 | int cpu = cpu_of(rq); |
Paul Turner | 67e8625 | 2010-11-15 15:47:05 -0800 | [diff] [blame] | 306 | /* |
| 307 | * Ensure we either appear before our parent (if already |
| 308 | * enqueued) or force our parent to appear after us when it is |
Vincent Guittot | 9c2791f | 2016-11-08 10:53:43 +0100 | [diff] [blame] | 309 | * enqueued. The fact that we always enqueue bottom-up |
| 310 | * reduces this to two cases and a special case for the root |
| 311 | * cfs_rq. Furthermore, it also means that we will always reset |
| 312 | * tmp_alone_branch either when the branch is connected |
| 313 | * to a tree or when we reach the beg of the tree |
Paul Turner | 67e8625 | 2010-11-15 15:47:05 -0800 | [diff] [blame] | 314 | */ |
| 315 | if (cfs_rq->tg->parent && |
Vincent Guittot | 9c2791f | 2016-11-08 10:53:43 +0100 | [diff] [blame] | 316 | cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { |
| 317 | /* |
| 318 | * If parent is already on the list, we add the child |
| 319 | * just before. Thanks to circular linked property of |
| 320 | * the list, this means to put the child at the tail |
| 321 | * of the list that starts by parent. |
| 322 | */ |
Paul Turner | 67e8625 | 2010-11-15 15:47:05 -0800 | [diff] [blame] | 323 | list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, |
Vincent Guittot | 9c2791f | 2016-11-08 10:53:43 +0100 | [diff] [blame] | 324 | &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); |
| 325 | /* |
| 326 | * The branch is now connected to its tree so we can |
| 327 | * reset tmp_alone_branch to the beginning of the |
| 328 | * list. |
| 329 | */ |
| 330 | rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; |
| 331 | } else if (!cfs_rq->tg->parent) { |
| 332 | /* |
| 333 | * cfs rq without parent should be put |
| 334 | * at the tail of the list. |
| 335 | */ |
| 336 | list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, |
| 337 | &rq->leaf_cfs_rq_list); |
| 338 | /* |
| 339 | * We have reach the beg of a tree so we can reset |
| 340 | * tmp_alone_branch to the beginning of the list. |
| 341 | */ |
| 342 | rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; |
| 343 | } else { |
| 344 | /* |
| 345 | * The parent has not already been added so we want to |
| 346 | * make sure that it will be put after us. |
| 347 | * tmp_alone_branch points to the beg of the branch |
| 348 | * where we will add parent. |
| 349 | */ |
| 350 | list_add_rcu(&cfs_rq->leaf_cfs_rq_list, |
| 351 | rq->tmp_alone_branch); |
| 352 | /* |
| 353 | * update tmp_alone_branch to points to the new beg |
| 354 | * of the branch |
| 355 | */ |
| 356 | rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; |
Paul Turner | 67e8625 | 2010-11-15 15:47:05 -0800 | [diff] [blame] | 357 | } |
Peter Zijlstra | 3d4b47b | 2010-11-15 15:47:01 -0800 | [diff] [blame] | 358 | |
| 359 | cfs_rq->on_list = 1; |
| 360 | } |
| 361 | } |
| 362 | |
| 363 | static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) |
| 364 | { |
| 365 | if (cfs_rq->on_list) { |
| 366 | list_del_rcu(&cfs_rq->leaf_cfs_rq_list); |
| 367 | cfs_rq->on_list = 0; |
| 368 | } |
| 369 | } |
| 370 | |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 371 | /* Iterate thr' all leaf cfs_rq's on a runqueue */ |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 372 | #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ |
| 373 | list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ |
| 374 | leaf_cfs_rq_list) |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 375 | |
| 376 | /* Do the two (enqueued) entities belong to the same group ? */ |
Peter Zijlstra | fed14d4 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 377 | static inline struct cfs_rq * |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 378 | is_same_group(struct sched_entity *se, struct sched_entity *pse) |
| 379 | { |
| 380 | if (se->cfs_rq == pse->cfs_rq) |
Peter Zijlstra | fed14d4 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 381 | return se->cfs_rq; |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 382 | |
Peter Zijlstra | fed14d4 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 383 | return NULL; |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 384 | } |
| 385 | |
| 386 | static inline struct sched_entity *parent_entity(struct sched_entity *se) |
| 387 | { |
| 388 | return se->parent; |
| 389 | } |
| 390 | |
Peter Zijlstra | 464b752 | 2008-10-24 11:06:15 +0200 | [diff] [blame] | 391 | static void |
| 392 | find_matching_se(struct sched_entity **se, struct sched_entity **pse) |
| 393 | { |
| 394 | int se_depth, pse_depth; |
| 395 | |
| 396 | /* |
| 397 | * preemption test can be made between sibling entities who are in the |
| 398 | * same cfs_rq i.e who have a common parent. Walk up the hierarchy of |
| 399 | * both tasks until we find their ancestors who are siblings of common |
| 400 | * parent. |
| 401 | */ |
| 402 | |
| 403 | /* First walk up until both entities are at same depth */ |
Peter Zijlstra | fed14d4 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 404 | se_depth = (*se)->depth; |
| 405 | pse_depth = (*pse)->depth; |
Peter Zijlstra | 464b752 | 2008-10-24 11:06:15 +0200 | [diff] [blame] | 406 | |
| 407 | while (se_depth > pse_depth) { |
| 408 | se_depth--; |
| 409 | *se = parent_entity(*se); |
| 410 | } |
| 411 | |
| 412 | while (pse_depth > se_depth) { |
| 413 | pse_depth--; |
| 414 | *pse = parent_entity(*pse); |
| 415 | } |
| 416 | |
| 417 | while (!is_same_group(*se, *pse)) { |
| 418 | *se = parent_entity(*se); |
| 419 | *pse = parent_entity(*pse); |
| 420 | } |
| 421 | } |
| 422 | |
Peter Zijlstra | 8f48894 | 2009-07-24 12:25:30 +0200 | [diff] [blame] | 423 | #else /* !CONFIG_FAIR_GROUP_SCHED */ |
| 424 | |
| 425 | static inline struct task_struct *task_of(struct sched_entity *se) |
| 426 | { |
| 427 | return container_of(se, struct task_struct, se); |
| 428 | } |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 429 | |
| 430 | static inline struct rq *rq_of(struct cfs_rq *cfs_rq) |
| 431 | { |
| 432 | return container_of(cfs_rq, struct rq, cfs); |
| 433 | } |
| 434 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 435 | #define entity_is_task(se) 1 |
| 436 | |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 437 | #define for_each_sched_entity(se) \ |
| 438 | for (; se; se = NULL) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 439 | |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 440 | static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 441 | { |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 442 | return &task_rq(p)->cfs; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 443 | } |
| 444 | |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 445 | static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) |
| 446 | { |
| 447 | struct task_struct *p = task_of(se); |
| 448 | struct rq *rq = task_rq(p); |
| 449 | |
| 450 | return &rq->cfs; |
| 451 | } |
| 452 | |
| 453 | /* runqueue "owned" by this group */ |
| 454 | static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) |
| 455 | { |
| 456 | return NULL; |
| 457 | } |
| 458 | |
Peter Zijlstra | 3d4b47b | 2010-11-15 15:47:01 -0800 | [diff] [blame] | 459 | static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) |
| 460 | { |
| 461 | } |
| 462 | |
| 463 | static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) |
| 464 | { |
| 465 | } |
| 466 | |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 467 | #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ |
| 468 | for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 469 | |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 470 | static inline struct sched_entity *parent_entity(struct sched_entity *se) |
| 471 | { |
| 472 | return NULL; |
| 473 | } |
| 474 | |
Peter Zijlstra | 464b752 | 2008-10-24 11:06:15 +0200 | [diff] [blame] | 475 | static inline void |
| 476 | find_matching_se(struct sched_entity **se, struct sched_entity **pse) |
| 477 | { |
| 478 | } |
| 479 | |
Peter Zijlstra | b758149 | 2008-04-19 19:45:00 +0200 | [diff] [blame] | 480 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
| 481 | |
Peter Zijlstra | 6c16a6d | 2012-03-21 13:07:16 -0700 | [diff] [blame] | 482 | static __always_inline |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 483 | void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 484 | |
| 485 | /************************************************************** |
| 486 | * Scheduling class tree data structure manipulation methods: |
| 487 | */ |
| 488 | |
Andrei Epure | 1bf0823 | 2013-03-12 21:12:24 +0200 | [diff] [blame] | 489 | static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) |
Peter Zijlstra | 02e0431 | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 490 | { |
Andrei Epure | 1bf0823 | 2013-03-12 21:12:24 +0200 | [diff] [blame] | 491 | s64 delta = (s64)(vruntime - max_vruntime); |
Peter Zijlstra | 368059a | 2007-10-15 17:00:11 +0200 | [diff] [blame] | 492 | if (delta > 0) |
Andrei Epure | 1bf0823 | 2013-03-12 21:12:24 +0200 | [diff] [blame] | 493 | max_vruntime = vruntime; |
Peter Zijlstra | 02e0431 | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 494 | |
Andrei Epure | 1bf0823 | 2013-03-12 21:12:24 +0200 | [diff] [blame] | 495 | return max_vruntime; |
Peter Zijlstra | 02e0431 | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 496 | } |
| 497 | |
Ingo Molnar | 0702e3e | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 498 | static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) |
Peter Zijlstra | b0ffd24 | 2007-10-15 17:00:12 +0200 | [diff] [blame] | 499 | { |
| 500 | s64 delta = (s64)(vruntime - min_vruntime); |
| 501 | if (delta < 0) |
| 502 | min_vruntime = vruntime; |
| 503 | |
| 504 | return min_vruntime; |
| 505 | } |
| 506 | |
Fabio Checconi | 54fdc58 | 2009-07-16 12:32:27 +0200 | [diff] [blame] | 507 | static inline int entity_before(struct sched_entity *a, |
| 508 | struct sched_entity *b) |
| 509 | { |
| 510 | return (s64)(a->vruntime - b->vruntime) < 0; |
| 511 | } |
| 512 | |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 513 | static void update_min_vruntime(struct cfs_rq *cfs_rq) |
| 514 | { |
Peter Zijlstra | b60205c | 2016-09-20 21:58:12 +0200 | [diff] [blame] | 515 | struct sched_entity *curr = cfs_rq->curr; |
| 516 | |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 517 | u64 vruntime = cfs_rq->min_vruntime; |
| 518 | |
Peter Zijlstra | b60205c | 2016-09-20 21:58:12 +0200 | [diff] [blame] | 519 | if (curr) { |
| 520 | if (curr->on_rq) |
| 521 | vruntime = curr->vruntime; |
| 522 | else |
| 523 | curr = NULL; |
| 524 | } |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 525 | |
| 526 | if (cfs_rq->rb_leftmost) { |
| 527 | struct sched_entity *se = rb_entry(cfs_rq->rb_leftmost, |
| 528 | struct sched_entity, |
| 529 | run_node); |
| 530 | |
Peter Zijlstra | b60205c | 2016-09-20 21:58:12 +0200 | [diff] [blame] | 531 | if (!curr) |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 532 | vruntime = se->vruntime; |
| 533 | else |
| 534 | vruntime = min_vruntime(vruntime, se->vruntime); |
| 535 | } |
| 536 | |
Andrei Epure | 1bf0823 | 2013-03-12 21:12:24 +0200 | [diff] [blame] | 537 | /* ensure we never gain time by being placed backwards. */ |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 538 | cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); |
Peter Zijlstra | 3fe1698 | 2011-04-05 17:23:48 +0200 | [diff] [blame] | 539 | #ifndef CONFIG_64BIT |
| 540 | smp_wmb(); |
| 541 | cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; |
| 542 | #endif |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 543 | } |
| 544 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 545 | /* |
| 546 | * Enqueue an entity into the rb-tree: |
| 547 | */ |
Ingo Molnar | 0702e3e | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 548 | static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 549 | { |
| 550 | struct rb_node **link = &cfs_rq->tasks_timeline.rb_node; |
| 551 | struct rb_node *parent = NULL; |
| 552 | struct sched_entity *entry; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 553 | int leftmost = 1; |
| 554 | |
| 555 | /* |
| 556 | * Find the right place in the rbtree: |
| 557 | */ |
| 558 | while (*link) { |
| 559 | parent = *link; |
| 560 | entry = rb_entry(parent, struct sched_entity, run_node); |
| 561 | /* |
| 562 | * We dont care about collisions. Nodes with |
| 563 | * the same key stay together. |
| 564 | */ |
Stephan Baerwolf | 2bd2d6f | 2011-07-20 14:46:59 +0200 | [diff] [blame] | 565 | if (entity_before(se, entry)) { |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 566 | link = &parent->rb_left; |
| 567 | } else { |
| 568 | link = &parent->rb_right; |
| 569 | leftmost = 0; |
| 570 | } |
| 571 | } |
| 572 | |
| 573 | /* |
| 574 | * Maintain a cache of leftmost tree entries (it is frequently |
| 575 | * used): |
| 576 | */ |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 577 | if (leftmost) |
Ingo Molnar | 57cb499 | 2007-10-15 17:00:11 +0200 | [diff] [blame] | 578 | cfs_rq->rb_leftmost = &se->run_node; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 579 | |
| 580 | rb_link_node(&se->run_node, parent, link); |
| 581 | rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 582 | } |
| 583 | |
Ingo Molnar | 0702e3e | 2007-10-15 17:00:14 +0200 | [diff] [blame] | 584 | static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 585 | { |
Peter Zijlstra | 3fe6974 | 2008-03-14 20:55:51 +0100 | [diff] [blame] | 586 | if (cfs_rq->rb_leftmost == &se->run_node) { |
| 587 | struct rb_node *next_node; |
Peter Zijlstra | 3fe6974 | 2008-03-14 20:55:51 +0100 | [diff] [blame] | 588 | |
| 589 | next_node = rb_next(&se->run_node); |
| 590 | cfs_rq->rb_leftmost = next_node; |
Peter Zijlstra | 3fe6974 | 2008-03-14 20:55:51 +0100 | [diff] [blame] | 591 | } |
Ingo Molnar | e9acbff | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 592 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 593 | rb_erase(&se->run_node, &cfs_rq->tasks_timeline); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 594 | } |
| 595 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 596 | struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 597 | { |
Peter Zijlstra | f4b6755 | 2008-11-04 21:25:07 +0100 | [diff] [blame] | 598 | struct rb_node *left = cfs_rq->rb_leftmost; |
| 599 | |
| 600 | if (!left) |
| 601 | return NULL; |
| 602 | |
| 603 | return rb_entry(left, struct sched_entity, run_node); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 604 | } |
| 605 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 606 | static struct sched_entity *__pick_next_entity(struct sched_entity *se) |
| 607 | { |
| 608 | struct rb_node *next = rb_next(&se->run_node); |
| 609 | |
| 610 | if (!next) |
| 611 | return NULL; |
| 612 | |
| 613 | return rb_entry(next, struct sched_entity, run_node); |
| 614 | } |
| 615 | |
| 616 | #ifdef CONFIG_SCHED_DEBUG |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 617 | struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 618 | { |
Ingo Molnar | 7eee3e6 | 2008-02-22 10:32:21 +0100 | [diff] [blame] | 619 | struct rb_node *last = rb_last(&cfs_rq->tasks_timeline); |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 620 | |
Balbir Singh | 70eee74 | 2008-02-22 13:25:53 +0530 | [diff] [blame] | 621 | if (!last) |
| 622 | return NULL; |
Ingo Molnar | 7eee3e6 | 2008-02-22 10:32:21 +0100 | [diff] [blame] | 623 | |
| 624 | return rb_entry(last, struct sched_entity, run_node); |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 625 | } |
| 626 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 627 | /************************************************************** |
| 628 | * Scheduling class statistics methods: |
| 629 | */ |
| 630 | |
Christian Ehrhardt | acb4a84 | 2009-11-30 12:16:48 +0100 | [diff] [blame] | 631 | int sched_proc_update_handler(struct ctl_table *table, int write, |
Alexey Dobriyan | 8d65af7 | 2009-09-23 15:57:19 -0700 | [diff] [blame] | 632 | void __user *buffer, size_t *lenp, |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 633 | loff_t *ppos) |
| 634 | { |
Alexey Dobriyan | 8d65af7 | 2009-09-23 15:57:19 -0700 | [diff] [blame] | 635 | int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); |
Nicholas Mc Guire | 58ac93e | 2015-05-15 21:05:42 +0200 | [diff] [blame] | 636 | unsigned int factor = get_update_sysctl_factor(); |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 637 | |
| 638 | if (ret || !write) |
| 639 | return ret; |
| 640 | |
| 641 | sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, |
| 642 | sysctl_sched_min_granularity); |
| 643 | |
Christian Ehrhardt | acb4a84 | 2009-11-30 12:16:48 +0100 | [diff] [blame] | 644 | #define WRT_SYSCTL(name) \ |
| 645 | (normalized_sysctl_##name = sysctl_##name / (factor)) |
| 646 | WRT_SYSCTL(sched_min_granularity); |
| 647 | WRT_SYSCTL(sched_latency); |
| 648 | WRT_SYSCTL(sched_wakeup_granularity); |
Christian Ehrhardt | acb4a84 | 2009-11-30 12:16:48 +0100 | [diff] [blame] | 649 | #undef WRT_SYSCTL |
| 650 | |
Peter Zijlstra | b2be5e9 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 651 | return 0; |
| 652 | } |
| 653 | #endif |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 654 | |
| 655 | /* |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 656 | * delta /= w |
Peter Zijlstra | a7be37a | 2008-06-27 13:41:11 +0200 | [diff] [blame] | 657 | */ |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 658 | static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) |
Peter Zijlstra | a7be37a | 2008-06-27 13:41:11 +0200 | [diff] [blame] | 659 | { |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 660 | if (unlikely(se->load.weight != NICE_0_LOAD)) |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 661 | delta = __calc_delta(delta, NICE_0_LOAD, &se->load); |
Peter Zijlstra | a7be37a | 2008-06-27 13:41:11 +0200 | [diff] [blame] | 662 | |
| 663 | return delta; |
| 664 | } |
| 665 | |
| 666 | /* |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 667 | * The idea is to set a period in which each task runs once. |
| 668 | * |
Borislav Petkov | 532b185 | 2012-08-08 16:16:04 +0200 | [diff] [blame] | 669 | * When there are too many tasks (sched_nr_latency) we have to stretch |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 670 | * this period because otherwise the slices get too small. |
| 671 | * |
| 672 | * p = (nr <= nl) ? l : l*nr/nl |
| 673 | */ |
Peter Zijlstra | 4d78e7b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 674 | static u64 __sched_period(unsigned long nr_running) |
| 675 | { |
Boqun Feng | 8e2b0bf | 2015-07-02 22:25:52 +0800 | [diff] [blame] | 676 | if (unlikely(nr_running > sched_nr_latency)) |
| 677 | return nr_running * sysctl_sched_min_granularity; |
| 678 | else |
| 679 | return sysctl_sched_latency; |
Peter Zijlstra | 4d78e7b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 680 | } |
| 681 | |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 682 | /* |
| 683 | * We calculate the wall-time slice from the period by taking a part |
| 684 | * proportional to the weight. |
| 685 | * |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 686 | * s = p*P[w/rw] |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 687 | */ |
Peter Zijlstra | 6d0f0eb | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 688 | static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 689 | { |
Mike Galbraith | 0a58244 | 2009-01-02 12:16:42 +0100 | [diff] [blame] | 690 | u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq); |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 691 | |
Mike Galbraith | 0a58244 | 2009-01-02 12:16:42 +0100 | [diff] [blame] | 692 | for_each_sched_entity(se) { |
Lin Ming | 6272d68 | 2009-01-15 17:17:15 +0100 | [diff] [blame] | 693 | struct load_weight *load; |
Christian Engelmayer | 3104bf0 | 2009-06-16 10:35:12 +0200 | [diff] [blame] | 694 | struct load_weight lw; |
Lin Ming | 6272d68 | 2009-01-15 17:17:15 +0100 | [diff] [blame] | 695 | |
| 696 | cfs_rq = cfs_rq_of(se); |
| 697 | load = &cfs_rq->load; |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 698 | |
Mike Galbraith | 0a58244 | 2009-01-02 12:16:42 +0100 | [diff] [blame] | 699 | if (unlikely(!se->on_rq)) { |
Christian Engelmayer | 3104bf0 | 2009-06-16 10:35:12 +0200 | [diff] [blame] | 700 | lw = cfs_rq->load; |
Mike Galbraith | 0a58244 | 2009-01-02 12:16:42 +0100 | [diff] [blame] | 701 | |
| 702 | update_load_add(&lw, se->load.weight); |
| 703 | load = &lw; |
| 704 | } |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 705 | slice = __calc_delta(slice, se->load.weight, load); |
Mike Galbraith | 0a58244 | 2009-01-02 12:16:42 +0100 | [diff] [blame] | 706 | } |
| 707 | return slice; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 708 | } |
| 709 | |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 710 | /* |
Andrei Epure | 660cc00 | 2013-03-11 12:03:20 +0200 | [diff] [blame] | 711 | * We calculate the vruntime slice of a to-be-inserted task. |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 712 | * |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 713 | * vs = s/w |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 714 | */ |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 715 | static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | 647e7ca | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 716 | { |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 717 | return calc_delta_fair(sched_slice(cfs_rq, se), se); |
Peter Zijlstra | a7be37a | 2008-06-27 13:41:11 +0200 | [diff] [blame] | 718 | } |
| 719 | |
Alex Shi | a75cdaa | 2013-06-20 10:18:47 +0800 | [diff] [blame] | 720 | #ifdef CONFIG_SMP |
Peter Zijlstra | 283e2ed | 2017-04-11 11:08:42 +0200 | [diff] [blame] | 721 | |
| 722 | #include "sched-pelt.h" |
| 723 | |
Morten Rasmussen | 772bd008c | 2016-06-22 18:03:13 +0100 | [diff] [blame] | 724 | static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 725 | static unsigned long task_h_load(struct task_struct *p); |
| 726 | |
Yuyang Du | 540247f | 2015-07-15 08:04:39 +0800 | [diff] [blame] | 727 | /* Give new sched_entity start runnable values to heavy its load in infant time */ |
| 728 | void init_entity_runnable_average(struct sched_entity *se) |
Alex Shi | a75cdaa | 2013-06-20 10:18:47 +0800 | [diff] [blame] | 729 | { |
Yuyang Du | 540247f | 2015-07-15 08:04:39 +0800 | [diff] [blame] | 730 | struct sched_avg *sa = &se->avg; |
Alex Shi | a75cdaa | 2013-06-20 10:18:47 +0800 | [diff] [blame] | 731 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 732 | sa->last_update_time = 0; |
| 733 | /* |
| 734 | * sched_avg's period_contrib should be strictly less then 1024, so |
| 735 | * we give it 1023 to make sure it is almost a period (1024us), and |
| 736 | * will definitely be update (after enqueue). |
| 737 | */ |
| 738 | sa->period_contrib = 1023; |
Vincent Guittot | b5a9b34 | 2016-10-19 14:45:23 +0200 | [diff] [blame] | 739 | /* |
| 740 | * Tasks are intialized with full load to be seen as heavy tasks until |
| 741 | * they get a chance to stabilize to their real load level. |
| 742 | * Group entities are intialized with zero load to reflect the fact that |
| 743 | * nothing has been attached to the task group yet. |
| 744 | */ |
| 745 | if (entity_is_task(se)) |
| 746 | sa->load_avg = scale_load_down(se->load.weight); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 747 | sa->load_sum = sa->load_avg * LOAD_AVG_MAX; |
Yuyang Du | 2b8c41d | 2016-03-30 04:30:56 +0800 | [diff] [blame] | 748 | /* |
| 749 | * At this point, util_avg won't be used in select_task_rq_fair anyway |
| 750 | */ |
| 751 | sa->util_avg = 0; |
| 752 | sa->util_sum = 0; |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 753 | /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ |
Alex Shi | a75cdaa | 2013-06-20 10:18:47 +0800 | [diff] [blame] | 754 | } |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 755 | |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 756 | static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq); |
Vincent Guittot | df21791 | 2016-11-08 10:53:42 +0100 | [diff] [blame] | 757 | static void attach_entity_cfs_rq(struct sched_entity *se); |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 758 | |
Yuyang Du | 2b8c41d | 2016-03-30 04:30:56 +0800 | [diff] [blame] | 759 | /* |
| 760 | * With new tasks being created, their initial util_avgs are extrapolated |
| 761 | * based on the cfs_rq's current util_avg: |
| 762 | * |
| 763 | * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight |
| 764 | * |
| 765 | * However, in many cases, the above util_avg does not give a desired |
| 766 | * value. Moreover, the sum of the util_avgs may be divergent, such |
| 767 | * as when the series is a harmonic series. |
| 768 | * |
| 769 | * To solve this problem, we also cap the util_avg of successive tasks to |
| 770 | * only 1/2 of the left utilization budget: |
| 771 | * |
| 772 | * util_avg_cap = (1024 - cfs_rq->avg.util_avg) / 2^n |
| 773 | * |
| 774 | * where n denotes the nth task. |
| 775 | * |
| 776 | * For example, a simplest series from the beginning would be like: |
| 777 | * |
| 778 | * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... |
| 779 | * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... |
| 780 | * |
| 781 | * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) |
| 782 | * if util_avg > util_avg_cap. |
| 783 | */ |
| 784 | void post_init_entity_util_avg(struct sched_entity *se) |
| 785 | { |
| 786 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 787 | struct sched_avg *sa = &se->avg; |
Yuyang Du | 172895e | 2016-04-05 12:12:27 +0800 | [diff] [blame] | 788 | long cap = (long)(SCHED_CAPACITY_SCALE - cfs_rq->avg.util_avg) / 2; |
Yuyang Du | 2b8c41d | 2016-03-30 04:30:56 +0800 | [diff] [blame] | 789 | |
| 790 | if (cap > 0) { |
| 791 | if (cfs_rq->avg.util_avg != 0) { |
| 792 | sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; |
| 793 | sa->util_avg /= (cfs_rq->avg.load_avg + 1); |
| 794 | |
| 795 | if (sa->util_avg > cap) |
| 796 | sa->util_avg = cap; |
| 797 | } else { |
| 798 | sa->util_avg = cap; |
| 799 | } |
| 800 | sa->util_sum = sa->util_avg * LOAD_AVG_MAX; |
| 801 | } |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 802 | |
| 803 | if (entity_is_task(se)) { |
| 804 | struct task_struct *p = task_of(se); |
| 805 | if (p->sched_class != &fair_sched_class) { |
| 806 | /* |
| 807 | * For !fair tasks do: |
| 808 | * |
Viresh Kumar | 3a123bb | 2017-05-24 10:59:56 +0530 | [diff] [blame] | 809 | update_cfs_rq_load_avg(now, cfs_rq); |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 810 | attach_entity_load_avg(cfs_rq, se); |
| 811 | switched_from_fair(rq, p); |
| 812 | * |
| 813 | * such that the next switched_to_fair() has the |
| 814 | * expected state. |
| 815 | */ |
Vincent Guittot | df21791 | 2016-11-08 10:53:42 +0100 | [diff] [blame] | 816 | se->avg.last_update_time = cfs_rq_clock_task(cfs_rq); |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 817 | return; |
| 818 | } |
| 819 | } |
| 820 | |
Vincent Guittot | df21791 | 2016-11-08 10:53:42 +0100 | [diff] [blame] | 821 | attach_entity_cfs_rq(se); |
Yuyang Du | 2b8c41d | 2016-03-30 04:30:56 +0800 | [diff] [blame] | 822 | } |
| 823 | |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 824 | #else /* !CONFIG_SMP */ |
Yuyang Du | 540247f | 2015-07-15 08:04:39 +0800 | [diff] [blame] | 825 | void init_entity_runnable_average(struct sched_entity *se) |
Alex Shi | a75cdaa | 2013-06-20 10:18:47 +0800 | [diff] [blame] | 826 | { |
| 827 | } |
Yuyang Du | 2b8c41d | 2016-03-30 04:30:56 +0800 | [diff] [blame] | 828 | void post_init_entity_util_avg(struct sched_entity *se) |
| 829 | { |
| 830 | } |
Peter Zijlstra | 3d30544f | 2016-06-21 14:27:50 +0200 | [diff] [blame] | 831 | static void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) |
| 832 | { |
| 833 | } |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 834 | #endif /* CONFIG_SMP */ |
Alex Shi | a75cdaa | 2013-06-20 10:18:47 +0800 | [diff] [blame] | 835 | |
Peter Zijlstra | a7be37a | 2008-06-27 13:41:11 +0200 | [diff] [blame] | 836 | /* |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 837 | * Update the current task's runtime statistics. |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 838 | */ |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 839 | static void update_curr(struct cfs_rq *cfs_rq) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 840 | { |
Ingo Molnar | 429d43b | 2007-10-15 17:00:03 +0200 | [diff] [blame] | 841 | struct sched_entity *curr = cfs_rq->curr; |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 842 | u64 now = rq_clock_task(rq_of(cfs_rq)); |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 843 | u64 delta_exec; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 844 | |
| 845 | if (unlikely(!curr)) |
| 846 | return; |
| 847 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 848 | delta_exec = now - curr->exec_start; |
| 849 | if (unlikely((s64)delta_exec <= 0)) |
Peter Zijlstra | 34f28ec | 2008-12-16 08:45:31 +0100 | [diff] [blame] | 850 | return; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 851 | |
Ingo Molnar | 8ebc91d | 2007-10-15 17:00:03 +0200 | [diff] [blame] | 852 | curr->exec_start = now; |
Srivatsa Vaddagiri | d842de8 | 2007-12-02 20:04:49 +0100 | [diff] [blame] | 853 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 854 | schedstat_set(curr->statistics.exec_max, |
| 855 | max(delta_exec, curr->statistics.exec_max)); |
| 856 | |
| 857 | curr->sum_exec_runtime += delta_exec; |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 858 | schedstat_add(cfs_rq->exec_clock, delta_exec); |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 859 | |
| 860 | curr->vruntime += calc_delta_fair(delta_exec, curr); |
| 861 | update_min_vruntime(cfs_rq); |
| 862 | |
Srivatsa Vaddagiri | d842de8 | 2007-12-02 20:04:49 +0100 | [diff] [blame] | 863 | if (entity_is_task(curr)) { |
| 864 | struct task_struct *curtask = task_of(curr); |
| 865 | |
Ingo Molnar | f977bb4 | 2009-09-13 18:15:54 +0200 | [diff] [blame] | 866 | trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); |
Srivatsa Vaddagiri | d842de8 | 2007-12-02 20:04:49 +0100 | [diff] [blame] | 867 | cpuacct_charge(curtask, delta_exec); |
Frank Mayhar | f06febc | 2008-09-12 09:54:39 -0700 | [diff] [blame] | 868 | account_group_exec_runtime(curtask, delta_exec); |
Srivatsa Vaddagiri | d842de8 | 2007-12-02 20:04:49 +0100 | [diff] [blame] | 869 | } |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 870 | |
| 871 | account_cfs_rq_runtime(cfs_rq, delta_exec); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 872 | } |
| 873 | |
Stanislaw Gruszka | 6e99891 | 2014-11-12 16:58:44 +0100 | [diff] [blame] | 874 | static void update_curr_fair(struct rq *rq) |
| 875 | { |
| 876 | update_curr(cfs_rq_of(&rq->curr->se)); |
| 877 | } |
| 878 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 879 | static inline void |
Ingo Molnar | 5870db5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 880 | update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 881 | { |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 882 | u64 wait_start, prev_wait_start; |
| 883 | |
| 884 | if (!schedstat_enabled()) |
| 885 | return; |
| 886 | |
| 887 | wait_start = rq_clock(rq_of(cfs_rq)); |
| 888 | prev_wait_start = schedstat_val(se->statistics.wait_start); |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 889 | |
| 890 | if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) && |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 891 | likely(wait_start > prev_wait_start)) |
| 892 | wait_start -= prev_wait_start; |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 893 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 894 | schedstat_set(se->statistics.wait_start, wait_start); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 895 | } |
| 896 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 897 | static inline void |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 898 | update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 899 | { |
| 900 | struct task_struct *p; |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 901 | u64 delta; |
| 902 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 903 | if (!schedstat_enabled()) |
| 904 | return; |
| 905 | |
| 906 | delta = rq_clock(rq_of(cfs_rq)) - schedstat_val(se->statistics.wait_start); |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 907 | |
| 908 | if (entity_is_task(se)) { |
| 909 | p = task_of(se); |
| 910 | if (task_on_rq_migrating(p)) { |
| 911 | /* |
| 912 | * Preserve migrating task's wait time so wait_start |
| 913 | * time stamp can be adjusted to accumulate wait time |
| 914 | * prior to migration. |
| 915 | */ |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 916 | schedstat_set(se->statistics.wait_start, delta); |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 917 | return; |
| 918 | } |
| 919 | trace_sched_stat_wait(p, delta); |
| 920 | } |
| 921 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 922 | schedstat_set(se->statistics.wait_max, |
| 923 | max(schedstat_val(se->statistics.wait_max), delta)); |
| 924 | schedstat_inc(se->statistics.wait_count); |
| 925 | schedstat_add(se->statistics.wait_sum, delta); |
| 926 | schedstat_set(se->statistics.wait_start, 0); |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 927 | } |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 928 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 929 | static inline void |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 930 | update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 931 | { |
| 932 | struct task_struct *tsk = NULL; |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 933 | u64 sleep_start, block_start; |
| 934 | |
| 935 | if (!schedstat_enabled()) |
| 936 | return; |
| 937 | |
| 938 | sleep_start = schedstat_val(se->statistics.sleep_start); |
| 939 | block_start = schedstat_val(se->statistics.block_start); |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 940 | |
| 941 | if (entity_is_task(se)) |
| 942 | tsk = task_of(se); |
| 943 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 944 | if (sleep_start) { |
| 945 | u64 delta = rq_clock(rq_of(cfs_rq)) - sleep_start; |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 946 | |
| 947 | if ((s64)delta < 0) |
| 948 | delta = 0; |
| 949 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 950 | if (unlikely(delta > schedstat_val(se->statistics.sleep_max))) |
| 951 | schedstat_set(se->statistics.sleep_max, delta); |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 952 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 953 | schedstat_set(se->statistics.sleep_start, 0); |
| 954 | schedstat_add(se->statistics.sum_sleep_runtime, delta); |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 955 | |
| 956 | if (tsk) { |
| 957 | account_scheduler_latency(tsk, delta >> 10, 1); |
| 958 | trace_sched_stat_sleep(tsk, delta); |
| 959 | } |
| 960 | } |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 961 | if (block_start) { |
| 962 | u64 delta = rq_clock(rq_of(cfs_rq)) - block_start; |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 963 | |
| 964 | if ((s64)delta < 0) |
| 965 | delta = 0; |
| 966 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 967 | if (unlikely(delta > schedstat_val(se->statistics.block_max))) |
| 968 | schedstat_set(se->statistics.block_max, delta); |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 969 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 970 | schedstat_set(se->statistics.block_start, 0); |
| 971 | schedstat_add(se->statistics.sum_sleep_runtime, delta); |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 972 | |
| 973 | if (tsk) { |
| 974 | if (tsk->in_iowait) { |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 975 | schedstat_add(se->statistics.iowait_sum, delta); |
| 976 | schedstat_inc(se->statistics.iowait_count); |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 977 | trace_sched_stat_iowait(tsk, delta); |
| 978 | } |
| 979 | |
| 980 | trace_sched_stat_blocked(tsk, delta); |
| 981 | |
| 982 | /* |
| 983 | * Blocking time is in units of nanosecs, so shift by |
| 984 | * 20 to get a milliseconds-range estimation of the |
| 985 | * amount of time that the task spent sleeping: |
| 986 | */ |
| 987 | if (unlikely(prof_on == SLEEP_PROFILING)) { |
| 988 | profile_hits(SLEEP_PROFILING, |
| 989 | (void *)get_wchan(tsk), |
| 990 | delta >> 20); |
| 991 | } |
| 992 | account_scheduler_latency(tsk, delta >> 10, 0); |
| 993 | } |
| 994 | } |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 995 | } |
| 996 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 997 | /* |
| 998 | * Task is being enqueued - update stats: |
| 999 | */ |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 1000 | static inline void |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 1001 | update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1002 | { |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 1003 | if (!schedstat_enabled()) |
| 1004 | return; |
| 1005 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1006 | /* |
| 1007 | * Are we enqueueing a waiting task? (for current tasks |
| 1008 | * a dequeue/enqueue event is a NOP) |
| 1009 | */ |
Ingo Molnar | 429d43b | 2007-10-15 17:00:03 +0200 | [diff] [blame] | 1010 | if (se != cfs_rq->curr) |
Ingo Molnar | 5870db5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 1011 | update_stats_wait_start(cfs_rq, se); |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 1012 | |
| 1013 | if (flags & ENQUEUE_WAKEUP) |
| 1014 | update_stats_enqueue_sleeper(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1015 | } |
| 1016 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1017 | static inline void |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 1018 | update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1019 | { |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 1020 | |
| 1021 | if (!schedstat_enabled()) |
| 1022 | return; |
| 1023 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1024 | /* |
| 1025 | * Mark the end of the wait period if dequeueing a |
| 1026 | * waiting task: |
| 1027 | */ |
Ingo Molnar | 429d43b | 2007-10-15 17:00:03 +0200 | [diff] [blame] | 1028 | if (se != cfs_rq->curr) |
Ingo Molnar | 9ef0a96 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 1029 | update_stats_wait_end(cfs_rq, se); |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 1030 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 1031 | if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { |
| 1032 | struct task_struct *tsk = task_of(se); |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 1033 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 1034 | if (tsk->state & TASK_INTERRUPTIBLE) |
| 1035 | schedstat_set(se->statistics.sleep_start, |
| 1036 | rq_clock(rq_of(cfs_rq))); |
| 1037 | if (tsk->state & TASK_UNINTERRUPTIBLE) |
| 1038 | schedstat_set(se->statistics.block_start, |
| 1039 | rq_clock(rq_of(cfs_rq))); |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 1040 | } |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1041 | } |
| 1042 | |
| 1043 | /* |
| 1044 | * We are picking a new current task - update its stats: |
| 1045 | */ |
| 1046 | static inline void |
Ingo Molnar | 79303e9 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 1047 | update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1048 | { |
| 1049 | /* |
| 1050 | * We are starting a new run period: |
| 1051 | */ |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 1052 | se->exec_start = rq_clock_task(rq_of(cfs_rq)); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1053 | } |
| 1054 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1055 | /************************************************** |
| 1056 | * Scheduling class queueing methods: |
| 1057 | */ |
| 1058 | |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 1059 | #ifdef CONFIG_NUMA_BALANCING |
| 1060 | /* |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1061 | * Approximate time to scan a full NUMA task in ms. The task scan period is |
| 1062 | * calculated based on the tasks virtual memory size and |
| 1063 | * numa_balancing_scan_size. |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 1064 | */ |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1065 | unsigned int sysctl_numa_balancing_scan_period_min = 1000; |
| 1066 | unsigned int sysctl_numa_balancing_scan_period_max = 60000; |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 1067 | |
| 1068 | /* Portion of address space to scan in MB */ |
| 1069 | unsigned int sysctl_numa_balancing_scan_size = 256; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 1070 | |
Peter Zijlstra | 4b96a29 | 2012-10-25 14:16:47 +0200 | [diff] [blame] | 1071 | /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ |
| 1072 | unsigned int sysctl_numa_balancing_scan_delay = 1000; |
| 1073 | |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 1074 | struct numa_group { |
| 1075 | atomic_t refcount; |
| 1076 | |
| 1077 | spinlock_t lock; /* nr_tasks, tasks */ |
| 1078 | int nr_tasks; |
| 1079 | pid_t gid; |
| 1080 | int active_nodes; |
| 1081 | |
| 1082 | struct rcu_head rcu; |
| 1083 | unsigned long total_faults; |
| 1084 | unsigned long max_faults_cpu; |
| 1085 | /* |
| 1086 | * Faults_cpu is used to decide whether memory should move |
| 1087 | * towards the CPU. As a consequence, these stats are weighted |
| 1088 | * more by CPU use than by memory faults. |
| 1089 | */ |
| 1090 | unsigned long *faults_cpu; |
| 1091 | unsigned long faults[0]; |
| 1092 | }; |
| 1093 | |
| 1094 | static inline unsigned long group_faults_priv(struct numa_group *ng); |
| 1095 | static inline unsigned long group_faults_shared(struct numa_group *ng); |
| 1096 | |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1097 | static unsigned int task_nr_scan_windows(struct task_struct *p) |
| 1098 | { |
| 1099 | unsigned long rss = 0; |
| 1100 | unsigned long nr_scan_pages; |
| 1101 | |
| 1102 | /* |
| 1103 | * Calculations based on RSS as non-present and empty pages are skipped |
| 1104 | * by the PTE scanner and NUMA hinting faults should be trapped based |
| 1105 | * on resident pages |
| 1106 | */ |
| 1107 | nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); |
| 1108 | rss = get_mm_rss(p->mm); |
| 1109 | if (!rss) |
| 1110 | rss = nr_scan_pages; |
| 1111 | |
| 1112 | rss = round_up(rss, nr_scan_pages); |
| 1113 | return rss / nr_scan_pages; |
| 1114 | } |
| 1115 | |
| 1116 | /* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ |
| 1117 | #define MAX_SCAN_WINDOW 2560 |
| 1118 | |
| 1119 | static unsigned int task_scan_min(struct task_struct *p) |
| 1120 | { |
Jason Low | 316c1608d | 2015-04-28 13:00:20 -0700 | [diff] [blame] | 1121 | unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1122 | unsigned int scan, floor; |
| 1123 | unsigned int windows = 1; |
| 1124 | |
Kirill Tkhai | 6419265 | 2014-10-16 14:39:37 +0400 | [diff] [blame] | 1125 | if (scan_size < MAX_SCAN_WINDOW) |
| 1126 | windows = MAX_SCAN_WINDOW / scan_size; |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1127 | floor = 1000 / windows; |
| 1128 | |
| 1129 | scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); |
| 1130 | return max_t(unsigned int, floor, scan); |
| 1131 | } |
| 1132 | |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 1133 | static unsigned int task_scan_start(struct task_struct *p) |
| 1134 | { |
| 1135 | unsigned long smin = task_scan_min(p); |
| 1136 | unsigned long period = smin; |
| 1137 | |
| 1138 | /* Scale the maximum scan period with the amount of shared memory. */ |
| 1139 | if (p->numa_group) { |
| 1140 | struct numa_group *ng = p->numa_group; |
| 1141 | unsigned long shared = group_faults_shared(ng); |
| 1142 | unsigned long private = group_faults_priv(ng); |
| 1143 | |
| 1144 | period *= atomic_read(&ng->refcount); |
| 1145 | period *= shared + 1; |
| 1146 | period /= private + shared + 1; |
| 1147 | } |
| 1148 | |
| 1149 | return max(smin, period); |
| 1150 | } |
| 1151 | |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1152 | static unsigned int task_scan_max(struct task_struct *p) |
| 1153 | { |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 1154 | unsigned long smin = task_scan_min(p); |
| 1155 | unsigned long smax; |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1156 | |
| 1157 | /* Watch for min being lower than max due to floor calculations */ |
| 1158 | smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 1159 | |
| 1160 | /* Scale the maximum scan period with the amount of shared memory. */ |
| 1161 | if (p->numa_group) { |
| 1162 | struct numa_group *ng = p->numa_group; |
| 1163 | unsigned long shared = group_faults_shared(ng); |
| 1164 | unsigned long private = group_faults_priv(ng); |
| 1165 | unsigned long period = smax; |
| 1166 | |
| 1167 | period *= atomic_read(&ng->refcount); |
| 1168 | period *= shared + 1; |
| 1169 | period /= private + shared + 1; |
| 1170 | |
| 1171 | smax = max(smax, period); |
| 1172 | } |
| 1173 | |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 1174 | return max(smin, smax); |
| 1175 | } |
| 1176 | |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 1177 | static void account_numa_enqueue(struct rq *rq, struct task_struct *p) |
| 1178 | { |
| 1179 | rq->nr_numa_running += (p->numa_preferred_nid != -1); |
| 1180 | rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); |
| 1181 | } |
| 1182 | |
| 1183 | static void account_numa_dequeue(struct rq *rq, struct task_struct *p) |
| 1184 | { |
| 1185 | rq->nr_numa_running -= (p->numa_preferred_nid != -1); |
| 1186 | rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); |
| 1187 | } |
| 1188 | |
Rik van Riel | be1e4e7 | 2014-01-27 17:03:48 -0500 | [diff] [blame] | 1189 | /* Shared or private faults. */ |
| 1190 | #define NR_NUMA_HINT_FAULT_TYPES 2 |
| 1191 | |
| 1192 | /* Memory and CPU locality */ |
| 1193 | #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) |
| 1194 | |
| 1195 | /* Averaged statistics, and temporary buffers. */ |
| 1196 | #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) |
| 1197 | |
Mel Gorman | e29cf08 | 2013-10-07 11:29:22 +0100 | [diff] [blame] | 1198 | pid_t task_numa_group_id(struct task_struct *p) |
| 1199 | { |
| 1200 | return p->numa_group ? p->numa_group->gid : 0; |
| 1201 | } |
| 1202 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1203 | /* |
| 1204 | * The averaged statistics, shared & private, memory & cpu, |
| 1205 | * occupy the first half of the array. The second half of the |
| 1206 | * array is for current counters, which are averaged into the |
| 1207 | * first set by task_numa_placement. |
| 1208 | */ |
| 1209 | static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) |
Mel Gorman | ac8e895 | 2013-10-07 11:29:03 +0100 | [diff] [blame] | 1210 | { |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1211 | return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; |
Mel Gorman | ac8e895 | 2013-10-07 11:29:03 +0100 | [diff] [blame] | 1212 | } |
| 1213 | |
| 1214 | static inline unsigned long task_faults(struct task_struct *p, int nid) |
| 1215 | { |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1216 | if (!p->numa_faults) |
Mel Gorman | ac8e895 | 2013-10-07 11:29:03 +0100 | [diff] [blame] | 1217 | return 0; |
| 1218 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1219 | return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + |
| 1220 | p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; |
Mel Gorman | ac8e895 | 2013-10-07 11:29:03 +0100 | [diff] [blame] | 1221 | } |
| 1222 | |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1223 | static inline unsigned long group_faults(struct task_struct *p, int nid) |
| 1224 | { |
| 1225 | if (!p->numa_group) |
| 1226 | return 0; |
| 1227 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1228 | return p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 0)] + |
| 1229 | p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 1)]; |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1230 | } |
| 1231 | |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1232 | static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) |
| 1233 | { |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1234 | return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] + |
| 1235 | group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)]; |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1236 | } |
| 1237 | |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 1238 | static inline unsigned long group_faults_priv(struct numa_group *ng) |
| 1239 | { |
| 1240 | unsigned long faults = 0; |
| 1241 | int node; |
| 1242 | |
| 1243 | for_each_online_node(node) { |
| 1244 | faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; |
| 1245 | } |
| 1246 | |
| 1247 | return faults; |
| 1248 | } |
| 1249 | |
| 1250 | static inline unsigned long group_faults_shared(struct numa_group *ng) |
| 1251 | { |
| 1252 | unsigned long faults = 0; |
| 1253 | int node; |
| 1254 | |
| 1255 | for_each_online_node(node) { |
| 1256 | faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; |
| 1257 | } |
| 1258 | |
| 1259 | return faults; |
| 1260 | } |
| 1261 | |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1262 | /* |
| 1263 | * A node triggering more than 1/3 as many NUMA faults as the maximum is |
| 1264 | * considered part of a numa group's pseudo-interleaving set. Migrations |
| 1265 | * between these nodes are slowed down, to allow things to settle down. |
| 1266 | */ |
| 1267 | #define ACTIVE_NODE_FRACTION 3 |
| 1268 | |
| 1269 | static bool numa_is_active_node(int nid, struct numa_group *ng) |
| 1270 | { |
| 1271 | return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; |
| 1272 | } |
| 1273 | |
Rik van Riel | 6c6b119 | 2014-10-17 03:29:52 -0400 | [diff] [blame] | 1274 | /* Handle placement on systems where not all nodes are directly connected. */ |
| 1275 | static unsigned long score_nearby_nodes(struct task_struct *p, int nid, |
| 1276 | int maxdist, bool task) |
| 1277 | { |
| 1278 | unsigned long score = 0; |
| 1279 | int node; |
| 1280 | |
| 1281 | /* |
| 1282 | * All nodes are directly connected, and the same distance |
| 1283 | * from each other. No need for fancy placement algorithms. |
| 1284 | */ |
| 1285 | if (sched_numa_topology_type == NUMA_DIRECT) |
| 1286 | return 0; |
| 1287 | |
| 1288 | /* |
| 1289 | * This code is called for each node, introducing N^2 complexity, |
| 1290 | * which should be ok given the number of nodes rarely exceeds 8. |
| 1291 | */ |
| 1292 | for_each_online_node(node) { |
| 1293 | unsigned long faults; |
| 1294 | int dist = node_distance(nid, node); |
| 1295 | |
| 1296 | /* |
| 1297 | * The furthest away nodes in the system are not interesting |
| 1298 | * for placement; nid was already counted. |
| 1299 | */ |
| 1300 | if (dist == sched_max_numa_distance || node == nid) |
| 1301 | continue; |
| 1302 | |
| 1303 | /* |
| 1304 | * On systems with a backplane NUMA topology, compare groups |
| 1305 | * of nodes, and move tasks towards the group with the most |
| 1306 | * memory accesses. When comparing two nodes at distance |
| 1307 | * "hoplimit", only nodes closer by than "hoplimit" are part |
| 1308 | * of each group. Skip other nodes. |
| 1309 | */ |
| 1310 | if (sched_numa_topology_type == NUMA_BACKPLANE && |
| 1311 | dist > maxdist) |
| 1312 | continue; |
| 1313 | |
| 1314 | /* Add up the faults from nearby nodes. */ |
| 1315 | if (task) |
| 1316 | faults = task_faults(p, node); |
| 1317 | else |
| 1318 | faults = group_faults(p, node); |
| 1319 | |
| 1320 | /* |
| 1321 | * On systems with a glueless mesh NUMA topology, there are |
| 1322 | * no fixed "groups of nodes". Instead, nodes that are not |
| 1323 | * directly connected bounce traffic through intermediate |
| 1324 | * nodes; a numa_group can occupy any set of nodes. |
| 1325 | * The further away a node is, the less the faults count. |
| 1326 | * This seems to result in good task placement. |
| 1327 | */ |
| 1328 | if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { |
| 1329 | faults *= (sched_max_numa_distance - dist); |
| 1330 | faults /= (sched_max_numa_distance - LOCAL_DISTANCE); |
| 1331 | } |
| 1332 | |
| 1333 | score += faults; |
| 1334 | } |
| 1335 | |
| 1336 | return score; |
| 1337 | } |
| 1338 | |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1339 | /* |
| 1340 | * These return the fraction of accesses done by a particular task, or |
| 1341 | * task group, on a particular numa node. The group weight is given a |
| 1342 | * larger multiplier, in order to group tasks together that are almost |
| 1343 | * evenly spread out between numa nodes. |
| 1344 | */ |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1345 | static inline unsigned long task_weight(struct task_struct *p, int nid, |
| 1346 | int dist) |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1347 | { |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1348 | unsigned long faults, total_faults; |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1349 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1350 | if (!p->numa_faults) |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1351 | return 0; |
| 1352 | |
| 1353 | total_faults = p->total_numa_faults; |
| 1354 | |
| 1355 | if (!total_faults) |
| 1356 | return 0; |
| 1357 | |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1358 | faults = task_faults(p, nid); |
Rik van Riel | 6c6b119 | 2014-10-17 03:29:52 -0400 | [diff] [blame] | 1359 | faults += score_nearby_nodes(p, nid, dist, true); |
| 1360 | |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1361 | return 1000 * faults / total_faults; |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1362 | } |
| 1363 | |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1364 | static inline unsigned long group_weight(struct task_struct *p, int nid, |
| 1365 | int dist) |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1366 | { |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1367 | unsigned long faults, total_faults; |
| 1368 | |
| 1369 | if (!p->numa_group) |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1370 | return 0; |
| 1371 | |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1372 | total_faults = p->numa_group->total_faults; |
| 1373 | |
| 1374 | if (!total_faults) |
| 1375 | return 0; |
| 1376 | |
| 1377 | faults = group_faults(p, nid); |
Rik van Riel | 6c6b119 | 2014-10-17 03:29:52 -0400 | [diff] [blame] | 1378 | faults += score_nearby_nodes(p, nid, dist, false); |
| 1379 | |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1380 | return 1000 * faults / total_faults; |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1381 | } |
| 1382 | |
Rik van Riel | 10f3904 | 2014-01-27 17:03:44 -0500 | [diff] [blame] | 1383 | bool should_numa_migrate_memory(struct task_struct *p, struct page * page, |
| 1384 | int src_nid, int dst_cpu) |
| 1385 | { |
| 1386 | struct numa_group *ng = p->numa_group; |
| 1387 | int dst_nid = cpu_to_node(dst_cpu); |
| 1388 | int last_cpupid, this_cpupid; |
| 1389 | |
| 1390 | this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); |
| 1391 | |
| 1392 | /* |
| 1393 | * Multi-stage node selection is used in conjunction with a periodic |
| 1394 | * migration fault to build a temporal task<->page relation. By using |
| 1395 | * a two-stage filter we remove short/unlikely relations. |
| 1396 | * |
| 1397 | * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate |
| 1398 | * a task's usage of a particular page (n_p) per total usage of this |
| 1399 | * page (n_t) (in a given time-span) to a probability. |
| 1400 | * |
| 1401 | * Our periodic faults will sample this probability and getting the |
| 1402 | * same result twice in a row, given these samples are fully |
| 1403 | * independent, is then given by P(n)^2, provided our sample period |
| 1404 | * is sufficiently short compared to the usage pattern. |
| 1405 | * |
| 1406 | * This quadric squishes small probabilities, making it less likely we |
| 1407 | * act on an unlikely task<->page relation. |
| 1408 | */ |
| 1409 | last_cpupid = page_cpupid_xchg_last(page, this_cpupid); |
| 1410 | if (!cpupid_pid_unset(last_cpupid) && |
| 1411 | cpupid_to_nid(last_cpupid) != dst_nid) |
| 1412 | return false; |
| 1413 | |
| 1414 | /* Always allow migrate on private faults */ |
| 1415 | if (cpupid_match_pid(p, last_cpupid)) |
| 1416 | return true; |
| 1417 | |
| 1418 | /* A shared fault, but p->numa_group has not been set up yet. */ |
| 1419 | if (!ng) |
| 1420 | return true; |
| 1421 | |
| 1422 | /* |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1423 | * Destination node is much more heavily used than the source |
| 1424 | * node? Allow migration. |
Rik van Riel | 10f3904 | 2014-01-27 17:03:44 -0500 | [diff] [blame] | 1425 | */ |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1426 | if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * |
| 1427 | ACTIVE_NODE_FRACTION) |
Rik van Riel | 10f3904 | 2014-01-27 17:03:44 -0500 | [diff] [blame] | 1428 | return true; |
| 1429 | |
| 1430 | /* |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1431 | * Distribute memory according to CPU & memory use on each node, |
| 1432 | * with 3/4 hysteresis to avoid unnecessary memory migrations: |
| 1433 | * |
| 1434 | * faults_cpu(dst) 3 faults_cpu(src) |
| 1435 | * --------------- * - > --------------- |
| 1436 | * faults_mem(dst) 4 faults_mem(src) |
Rik van Riel | 10f3904 | 2014-01-27 17:03:44 -0500 | [diff] [blame] | 1437 | */ |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1438 | return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > |
| 1439 | group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; |
Rik van Riel | 10f3904 | 2014-01-27 17:03:44 -0500 | [diff] [blame] | 1440 | } |
| 1441 | |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 1442 | static unsigned long weighted_cpuload(struct rq *rq); |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1443 | static unsigned long source_load(int cpu, int type); |
| 1444 | static unsigned long target_load(int cpu, int type); |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 1445 | static unsigned long capacity_of(int cpu); |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 1446 | |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1447 | /* Cached statistics for all CPUs within a node */ |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1448 | struct numa_stats { |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1449 | unsigned long nr_running; |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1450 | unsigned long load; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1451 | |
| 1452 | /* Total compute capacity of CPUs on a node */ |
Nicolas Pitre | 5ef20ca | 2014-05-26 18:19:34 -0400 | [diff] [blame] | 1453 | unsigned long compute_capacity; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1454 | |
| 1455 | /* Approximate capacity in terms of runnable tasks on a node */ |
Nicolas Pitre | 5ef20ca | 2014-05-26 18:19:34 -0400 | [diff] [blame] | 1456 | unsigned long task_capacity; |
Nicolas Pitre | 1b6a749 | 2014-05-26 18:19:35 -0400 | [diff] [blame] | 1457 | int has_free_capacity; |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1458 | }; |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 1459 | |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1460 | /* |
| 1461 | * XXX borrowed from update_sg_lb_stats |
| 1462 | */ |
| 1463 | static void update_numa_stats(struct numa_stats *ns, int nid) |
| 1464 | { |
Rik van Riel | 83d7f24 | 2014-08-04 13:23:28 -0400 | [diff] [blame] | 1465 | int smt, cpu, cpus = 0; |
| 1466 | unsigned long capacity; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1467 | |
| 1468 | memset(ns, 0, sizeof(*ns)); |
| 1469 | for_each_cpu(cpu, cpumask_of_node(nid)) { |
| 1470 | struct rq *rq = cpu_rq(cpu); |
| 1471 | |
| 1472 | ns->nr_running += rq->nr_running; |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 1473 | ns->load += weighted_cpuload(rq); |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 1474 | ns->compute_capacity += capacity_of(cpu); |
Peter Zijlstra | 5eca82a | 2013-11-06 18:47:57 +0100 | [diff] [blame] | 1475 | |
| 1476 | cpus++; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1477 | } |
| 1478 | |
Peter Zijlstra | 5eca82a | 2013-11-06 18:47:57 +0100 | [diff] [blame] | 1479 | /* |
| 1480 | * If we raced with hotplug and there are no CPUs left in our mask |
| 1481 | * the @ns structure is NULL'ed and task_numa_compare() will |
| 1482 | * not find this node attractive. |
| 1483 | * |
Nicolas Pitre | 1b6a749 | 2014-05-26 18:19:35 -0400 | [diff] [blame] | 1484 | * We'll either bail at !has_free_capacity, or we'll detect a huge |
| 1485 | * imbalance and bail there. |
Peter Zijlstra | 5eca82a | 2013-11-06 18:47:57 +0100 | [diff] [blame] | 1486 | */ |
| 1487 | if (!cpus) |
| 1488 | return; |
| 1489 | |
Rik van Riel | 83d7f24 | 2014-08-04 13:23:28 -0400 | [diff] [blame] | 1490 | /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */ |
| 1491 | smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity); |
| 1492 | capacity = cpus / smt; /* cores */ |
| 1493 | |
| 1494 | ns->task_capacity = min_t(unsigned, capacity, |
| 1495 | DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE)); |
Nicolas Pitre | 1b6a749 | 2014-05-26 18:19:35 -0400 | [diff] [blame] | 1496 | ns->has_free_capacity = (ns->nr_running < ns->task_capacity); |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1497 | } |
| 1498 | |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1499 | struct task_numa_env { |
| 1500 | struct task_struct *p; |
| 1501 | |
| 1502 | int src_cpu, src_nid; |
| 1503 | int dst_cpu, dst_nid; |
| 1504 | |
| 1505 | struct numa_stats src_stats, dst_stats; |
| 1506 | |
Wanpeng Li | 40ea2b4 | 2013-12-05 19:10:17 +0800 | [diff] [blame] | 1507 | int imbalance_pct; |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1508 | int dist; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1509 | |
| 1510 | struct task_struct *best_task; |
| 1511 | long best_imp; |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1512 | int best_cpu; |
| 1513 | }; |
| 1514 | |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1515 | static void task_numa_assign(struct task_numa_env *env, |
| 1516 | struct task_struct *p, long imp) |
| 1517 | { |
| 1518 | if (env->best_task) |
| 1519 | put_task_struct(env->best_task); |
Oleg Nesterov | bac7857 | 2016-05-18 21:57:33 +0200 | [diff] [blame] | 1520 | if (p) |
| 1521 | get_task_struct(p); |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1522 | |
| 1523 | env->best_task = p; |
| 1524 | env->best_imp = imp; |
| 1525 | env->best_cpu = env->dst_cpu; |
| 1526 | } |
| 1527 | |
Rik van Riel | 28a2174 | 2014-06-23 11:46:13 -0400 | [diff] [blame] | 1528 | static bool load_too_imbalanced(long src_load, long dst_load, |
Rik van Riel | e63da03 | 2014-05-14 13:22:21 -0400 | [diff] [blame] | 1529 | struct task_numa_env *env) |
| 1530 | { |
Rik van Riel | e4991b2 | 2015-05-27 15:04:27 -0400 | [diff] [blame] | 1531 | long imb, old_imb; |
| 1532 | long orig_src_load, orig_dst_load; |
Rik van Riel | 28a2174 | 2014-06-23 11:46:13 -0400 | [diff] [blame] | 1533 | long src_capacity, dst_capacity; |
| 1534 | |
| 1535 | /* |
| 1536 | * The load is corrected for the CPU capacity available on each node. |
| 1537 | * |
| 1538 | * src_load dst_load |
| 1539 | * ------------ vs --------- |
| 1540 | * src_capacity dst_capacity |
| 1541 | */ |
| 1542 | src_capacity = env->src_stats.compute_capacity; |
| 1543 | dst_capacity = env->dst_stats.compute_capacity; |
Rik van Riel | e63da03 | 2014-05-14 13:22:21 -0400 | [diff] [blame] | 1544 | |
| 1545 | /* We care about the slope of the imbalance, not the direction. */ |
Rik van Riel | e4991b2 | 2015-05-27 15:04:27 -0400 | [diff] [blame] | 1546 | if (dst_load < src_load) |
| 1547 | swap(dst_load, src_load); |
Rik van Riel | e63da03 | 2014-05-14 13:22:21 -0400 | [diff] [blame] | 1548 | |
| 1549 | /* Is the difference below the threshold? */ |
Rik van Riel | e4991b2 | 2015-05-27 15:04:27 -0400 | [diff] [blame] | 1550 | imb = dst_load * src_capacity * 100 - |
| 1551 | src_load * dst_capacity * env->imbalance_pct; |
Rik van Riel | e63da03 | 2014-05-14 13:22:21 -0400 | [diff] [blame] | 1552 | if (imb <= 0) |
| 1553 | return false; |
| 1554 | |
| 1555 | /* |
| 1556 | * The imbalance is above the allowed threshold. |
Rik van Riel | e4991b2 | 2015-05-27 15:04:27 -0400 | [diff] [blame] | 1557 | * Compare it with the old imbalance. |
Rik van Riel | e63da03 | 2014-05-14 13:22:21 -0400 | [diff] [blame] | 1558 | */ |
Rik van Riel | 28a2174 | 2014-06-23 11:46:13 -0400 | [diff] [blame] | 1559 | orig_src_load = env->src_stats.load; |
Rik van Riel | e4991b2 | 2015-05-27 15:04:27 -0400 | [diff] [blame] | 1560 | orig_dst_load = env->dst_stats.load; |
Rik van Riel | 28a2174 | 2014-06-23 11:46:13 -0400 | [diff] [blame] | 1561 | |
Rik van Riel | e4991b2 | 2015-05-27 15:04:27 -0400 | [diff] [blame] | 1562 | if (orig_dst_load < orig_src_load) |
| 1563 | swap(orig_dst_load, orig_src_load); |
Rik van Riel | e63da03 | 2014-05-14 13:22:21 -0400 | [diff] [blame] | 1564 | |
Rik van Riel | e4991b2 | 2015-05-27 15:04:27 -0400 | [diff] [blame] | 1565 | old_imb = orig_dst_load * src_capacity * 100 - |
| 1566 | orig_src_load * dst_capacity * env->imbalance_pct; |
| 1567 | |
| 1568 | /* Would this change make things worse? */ |
| 1569 | return (imb > old_imb); |
Rik van Riel | e63da03 | 2014-05-14 13:22:21 -0400 | [diff] [blame] | 1570 | } |
| 1571 | |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1572 | /* |
| 1573 | * This checks if the overall compute and NUMA accesses of the system would |
| 1574 | * be improved if the source tasks was migrated to the target dst_cpu taking |
| 1575 | * into account that it might be best if task running on the dst_cpu should |
| 1576 | * be exchanged with the source task |
| 1577 | */ |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1578 | static void task_numa_compare(struct task_numa_env *env, |
| 1579 | long taskimp, long groupimp) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1580 | { |
| 1581 | struct rq *src_rq = cpu_rq(env->src_cpu); |
| 1582 | struct rq *dst_rq = cpu_rq(env->dst_cpu); |
| 1583 | struct task_struct *cur; |
Rik van Riel | 28a2174 | 2014-06-23 11:46:13 -0400 | [diff] [blame] | 1584 | long src_load, dst_load; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1585 | long load; |
Rik van Riel | 1c5d3eb | 2014-06-23 11:46:15 -0400 | [diff] [blame] | 1586 | long imp = env->p->numa_group ? groupimp : taskimp; |
Rik van Riel | 0132c3e | 2014-06-23 11:46:16 -0400 | [diff] [blame] | 1587 | long moveimp = imp; |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1588 | int dist = env->dist; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1589 | |
| 1590 | rcu_read_lock(); |
Oleg Nesterov | bac7857 | 2016-05-18 21:57:33 +0200 | [diff] [blame] | 1591 | cur = task_rcu_dereference(&dst_rq->curr); |
| 1592 | if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur))) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1593 | cur = NULL; |
| 1594 | |
| 1595 | /* |
Peter Zijlstra | 7af6833 | 2014-11-10 10:54:35 +0100 | [diff] [blame] | 1596 | * Because we have preemption enabled we can get migrated around and |
| 1597 | * end try selecting ourselves (current == env->p) as a swap candidate. |
| 1598 | */ |
| 1599 | if (cur == env->p) |
| 1600 | goto unlock; |
| 1601 | |
| 1602 | /* |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1603 | * "imp" is the fault differential for the source task between the |
| 1604 | * source and destination node. Calculate the total differential for |
| 1605 | * the source task and potential destination task. The more negative |
| 1606 | * the value is, the more rmeote accesses that would be expected to |
| 1607 | * be incurred if the tasks were swapped. |
| 1608 | */ |
| 1609 | if (cur) { |
| 1610 | /* Skip this swap candidate if cannot move to the source cpu */ |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 1611 | if (!cpumask_test_cpu(env->src_cpu, &cur->cpus_allowed)) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1612 | goto unlock; |
| 1613 | |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1614 | /* |
| 1615 | * If dst and source tasks are in the same NUMA group, or not |
Rik van Riel | ca28aa53 | 2013-10-07 11:29:32 +0100 | [diff] [blame] | 1616 | * in any group then look only at task weights. |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1617 | */ |
Rik van Riel | ca28aa53 | 2013-10-07 11:29:32 +0100 | [diff] [blame] | 1618 | if (cur->numa_group == env->p->numa_group) { |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1619 | imp = taskimp + task_weight(cur, env->src_nid, dist) - |
| 1620 | task_weight(cur, env->dst_nid, dist); |
Rik van Riel | ca28aa53 | 2013-10-07 11:29:32 +0100 | [diff] [blame] | 1621 | /* |
| 1622 | * Add some hysteresis to prevent swapping the |
| 1623 | * tasks within a group over tiny differences. |
| 1624 | */ |
| 1625 | if (cur->numa_group) |
| 1626 | imp -= imp/16; |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1627 | } else { |
Rik van Riel | ca28aa53 | 2013-10-07 11:29:32 +0100 | [diff] [blame] | 1628 | /* |
| 1629 | * Compare the group weights. If a task is all by |
| 1630 | * itself (not part of a group), use the task weight |
| 1631 | * instead. |
| 1632 | */ |
Rik van Riel | ca28aa53 | 2013-10-07 11:29:32 +0100 | [diff] [blame] | 1633 | if (cur->numa_group) |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1634 | imp += group_weight(cur, env->src_nid, dist) - |
| 1635 | group_weight(cur, env->dst_nid, dist); |
Rik van Riel | ca28aa53 | 2013-10-07 11:29:32 +0100 | [diff] [blame] | 1636 | else |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1637 | imp += task_weight(cur, env->src_nid, dist) - |
| 1638 | task_weight(cur, env->dst_nid, dist); |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1639 | } |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1640 | } |
| 1641 | |
Rik van Riel | 0132c3e | 2014-06-23 11:46:16 -0400 | [diff] [blame] | 1642 | if (imp <= env->best_imp && moveimp <= env->best_imp) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1643 | goto unlock; |
| 1644 | |
| 1645 | if (!cur) { |
| 1646 | /* Is there capacity at our destination? */ |
Rik van Riel | b932c03 | 2014-08-04 13:23:27 -0400 | [diff] [blame] | 1647 | if (env->src_stats.nr_running <= env->src_stats.task_capacity && |
Nicolas Pitre | 1b6a749 | 2014-05-26 18:19:35 -0400 | [diff] [blame] | 1648 | !env->dst_stats.has_free_capacity) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1649 | goto unlock; |
| 1650 | |
| 1651 | goto balance; |
| 1652 | } |
| 1653 | |
| 1654 | /* Balance doesn't matter much if we're running a task per cpu */ |
Rik van Riel | 0132c3e | 2014-06-23 11:46:16 -0400 | [diff] [blame] | 1655 | if (imp > env->best_imp && src_rq->nr_running == 1 && |
| 1656 | dst_rq->nr_running == 1) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1657 | goto assign; |
| 1658 | |
| 1659 | /* |
| 1660 | * In the overloaded case, try and keep the load balanced. |
| 1661 | */ |
| 1662 | balance: |
Peter Zijlstra | e720fff | 2014-07-11 16:01:53 +0200 | [diff] [blame] | 1663 | load = task_h_load(env->p); |
| 1664 | dst_load = env->dst_stats.load + load; |
| 1665 | src_load = env->src_stats.load - load; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1666 | |
Rik van Riel | 0132c3e | 2014-06-23 11:46:16 -0400 | [diff] [blame] | 1667 | if (moveimp > imp && moveimp > env->best_imp) { |
| 1668 | /* |
| 1669 | * If the improvement from just moving env->p direction is |
| 1670 | * better than swapping tasks around, check if a move is |
| 1671 | * possible. Store a slightly smaller score than moveimp, |
| 1672 | * so an actually idle CPU will win. |
| 1673 | */ |
| 1674 | if (!load_too_imbalanced(src_load, dst_load, env)) { |
| 1675 | imp = moveimp - 1; |
| 1676 | cur = NULL; |
| 1677 | goto assign; |
| 1678 | } |
| 1679 | } |
| 1680 | |
| 1681 | if (imp <= env->best_imp) |
| 1682 | goto unlock; |
| 1683 | |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1684 | if (cur) { |
Peter Zijlstra | e720fff | 2014-07-11 16:01:53 +0200 | [diff] [blame] | 1685 | load = task_h_load(cur); |
| 1686 | dst_load -= load; |
| 1687 | src_load += load; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1688 | } |
| 1689 | |
Rik van Riel | 28a2174 | 2014-06-23 11:46:13 -0400 | [diff] [blame] | 1690 | if (load_too_imbalanced(src_load, dst_load, env)) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1691 | goto unlock; |
| 1692 | |
Rik van Riel | ba7e5a2 | 2014-09-04 16:35:30 -0400 | [diff] [blame] | 1693 | /* |
| 1694 | * One idle CPU per node is evaluated for a task numa move. |
| 1695 | * Call select_idle_sibling to maybe find a better one. |
| 1696 | */ |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 1697 | if (!cur) { |
| 1698 | /* |
| 1699 | * select_idle_siblings() uses an per-cpu cpumask that |
| 1700 | * can be used from IRQ context. |
| 1701 | */ |
| 1702 | local_irq_disable(); |
Morten Rasmussen | 772bd008c | 2016-06-22 18:03:13 +0100 | [diff] [blame] | 1703 | env->dst_cpu = select_idle_sibling(env->p, env->src_cpu, |
| 1704 | env->dst_cpu); |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 1705 | local_irq_enable(); |
| 1706 | } |
Rik van Riel | ba7e5a2 | 2014-09-04 16:35:30 -0400 | [diff] [blame] | 1707 | |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1708 | assign: |
| 1709 | task_numa_assign(env, cur, imp); |
| 1710 | unlock: |
| 1711 | rcu_read_unlock(); |
| 1712 | } |
| 1713 | |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1714 | static void task_numa_find_cpu(struct task_numa_env *env, |
| 1715 | long taskimp, long groupimp) |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1716 | { |
| 1717 | int cpu; |
| 1718 | |
| 1719 | for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { |
| 1720 | /* Skip this CPU if the source task cannot migrate */ |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 1721 | if (!cpumask_test_cpu(cpu, &env->p->cpus_allowed)) |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1722 | continue; |
| 1723 | |
| 1724 | env->dst_cpu = cpu; |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1725 | task_numa_compare(env, taskimp, groupimp); |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1726 | } |
| 1727 | } |
| 1728 | |
Rik van Riel | 6f9aad0 | 2015-05-28 09:52:49 -0400 | [diff] [blame] | 1729 | /* Only move tasks to a NUMA node less busy than the current node. */ |
| 1730 | static bool numa_has_capacity(struct task_numa_env *env) |
| 1731 | { |
| 1732 | struct numa_stats *src = &env->src_stats; |
| 1733 | struct numa_stats *dst = &env->dst_stats; |
| 1734 | |
| 1735 | if (src->has_free_capacity && !dst->has_free_capacity) |
| 1736 | return false; |
| 1737 | |
| 1738 | /* |
| 1739 | * Only consider a task move if the source has a higher load |
| 1740 | * than the destination, corrected for CPU capacity on each node. |
| 1741 | * |
| 1742 | * src->load dst->load |
| 1743 | * --------------------- vs --------------------- |
| 1744 | * src->compute_capacity dst->compute_capacity |
| 1745 | */ |
Srikar Dronamraju | 44dcb04 | 2015-06-16 17:26:00 +0530 | [diff] [blame] | 1746 | if (src->load * dst->compute_capacity * env->imbalance_pct > |
| 1747 | |
| 1748 | dst->load * src->compute_capacity * 100) |
Rik van Riel | 6f9aad0 | 2015-05-28 09:52:49 -0400 | [diff] [blame] | 1749 | return true; |
| 1750 | |
| 1751 | return false; |
| 1752 | } |
| 1753 | |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1754 | static int task_numa_migrate(struct task_struct *p) |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 1755 | { |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1756 | struct task_numa_env env = { |
| 1757 | .p = p, |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1758 | |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1759 | .src_cpu = task_cpu(p), |
Ingo Molnar | b32e86b | 2013-10-07 11:29:30 +0100 | [diff] [blame] | 1760 | .src_nid = task_node(p), |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1761 | |
| 1762 | .imbalance_pct = 112, |
| 1763 | |
| 1764 | .best_task = NULL, |
| 1765 | .best_imp = 0, |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1766 | .best_cpu = -1, |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1767 | }; |
| 1768 | struct sched_domain *sd; |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1769 | unsigned long taskweight, groupweight; |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1770 | int nid, ret, dist; |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1771 | long taskimp, groupimp; |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 1772 | |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1773 | /* |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1774 | * Pick the lowest SD_NUMA domain, as that would have the smallest |
| 1775 | * imbalance and would be the first to start moving tasks about. |
| 1776 | * |
| 1777 | * And we want to avoid any moving of tasks about, as that would create |
| 1778 | * random movement of tasks -- counter the numa conditions we're trying |
| 1779 | * to satisfy here. |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1780 | */ |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 1781 | rcu_read_lock(); |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1782 | sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); |
Rik van Riel | 46a73e8 | 2013-11-11 19:29:25 -0500 | [diff] [blame] | 1783 | if (sd) |
| 1784 | env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 1785 | rcu_read_unlock(); |
| 1786 | |
Rik van Riel | 46a73e8 | 2013-11-11 19:29:25 -0500 | [diff] [blame] | 1787 | /* |
| 1788 | * Cpusets can break the scheduler domain tree into smaller |
| 1789 | * balance domains, some of which do not cross NUMA boundaries. |
| 1790 | * Tasks that are "trapped" in such domains cannot be migrated |
| 1791 | * elsewhere, so there is no point in (re)trying. |
| 1792 | */ |
| 1793 | if (unlikely(!sd)) { |
Wanpeng Li | de1b301 | 2013-12-12 15:23:24 +0800 | [diff] [blame] | 1794 | p->numa_preferred_nid = task_node(p); |
Rik van Riel | 46a73e8 | 2013-11-11 19:29:25 -0500 | [diff] [blame] | 1795 | return -EINVAL; |
| 1796 | } |
| 1797 | |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1798 | env.dst_nid = p->numa_preferred_nid; |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1799 | dist = env.dist = node_distance(env.src_nid, env.dst_nid); |
| 1800 | taskweight = task_weight(p, env.src_nid, dist); |
| 1801 | groupweight = group_weight(p, env.src_nid, dist); |
| 1802 | update_numa_stats(&env.src_stats, env.src_nid); |
| 1803 | taskimp = task_weight(p, env.dst_nid, dist) - taskweight; |
| 1804 | groupimp = group_weight(p, env.dst_nid, dist) - groupweight; |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1805 | update_numa_stats(&env.dst_stats, env.dst_nid); |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1806 | |
Rik van Riel | a43455a | 2014-06-04 16:09:42 -0400 | [diff] [blame] | 1807 | /* Try to find a spot on the preferred nid. */ |
Rik van Riel | 6f9aad0 | 2015-05-28 09:52:49 -0400 | [diff] [blame] | 1808 | if (numa_has_capacity(&env)) |
| 1809 | task_numa_find_cpu(&env, taskimp, groupimp); |
Rik van Riel | e1dda8a | 2013-10-07 11:29:19 +0100 | [diff] [blame] | 1810 | |
Rik van Riel | 9de05d4 | 2014-10-09 17:27:47 -0400 | [diff] [blame] | 1811 | /* |
| 1812 | * Look at other nodes in these cases: |
| 1813 | * - there is no space available on the preferred_nid |
| 1814 | * - the task is part of a numa_group that is interleaved across |
| 1815 | * multiple NUMA nodes; in order to better consolidate the group, |
| 1816 | * we need to check other locations. |
| 1817 | */ |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1818 | if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) { |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1819 | for_each_online_node(nid) { |
| 1820 | if (nid == env.src_nid || nid == p->numa_preferred_nid) |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1821 | continue; |
| 1822 | |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1823 | dist = node_distance(env.src_nid, env.dst_nid); |
Rik van Riel | 6c6b119 | 2014-10-17 03:29:52 -0400 | [diff] [blame] | 1824 | if (sched_numa_topology_type == NUMA_BACKPLANE && |
| 1825 | dist != env.dist) { |
| 1826 | taskweight = task_weight(p, env.src_nid, dist); |
| 1827 | groupweight = group_weight(p, env.src_nid, dist); |
| 1828 | } |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1829 | |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 1830 | /* Only consider nodes where both task and groups benefit */ |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1831 | taskimp = task_weight(p, nid, dist) - taskweight; |
| 1832 | groupimp = group_weight(p, nid, dist) - groupweight; |
Rik van Riel | 887c290 | 2013-10-07 11:29:31 +0100 | [diff] [blame] | 1833 | if (taskimp < 0 && groupimp < 0) |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1834 | continue; |
| 1835 | |
Rik van Riel | 7bd9532 | 2014-10-17 03:29:51 -0400 | [diff] [blame] | 1836 | env.dist = dist; |
Mel Gorman | 2c8a50a | 2013-10-07 11:29:18 +0100 | [diff] [blame] | 1837 | env.dst_nid = nid; |
| 1838 | update_numa_stats(&env.dst_stats, env.dst_nid); |
Rik van Riel | 6f9aad0 | 2015-05-28 09:52:49 -0400 | [diff] [blame] | 1839 | if (numa_has_capacity(&env)) |
| 1840 | task_numa_find_cpu(&env, taskimp, groupimp); |
Mel Gorman | 58d081b | 2013-10-07 11:29:10 +0100 | [diff] [blame] | 1841 | } |
| 1842 | } |
| 1843 | |
Rik van Riel | 68d1b02 | 2014-04-11 13:00:29 -0400 | [diff] [blame] | 1844 | /* |
| 1845 | * If the task is part of a workload that spans multiple NUMA nodes, |
| 1846 | * and is migrating into one of the workload's active nodes, remember |
| 1847 | * this node as the task's preferred numa node, so the workload can |
| 1848 | * settle down. |
| 1849 | * A task that migrated to a second choice node will be better off |
| 1850 | * trying for a better one later. Do not set the preferred node here. |
| 1851 | */ |
Rik van Riel | db015da | 2014-06-23 11:41:34 -0400 | [diff] [blame] | 1852 | if (p->numa_group) { |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1853 | struct numa_group *ng = p->numa_group; |
| 1854 | |
Rik van Riel | db015da | 2014-06-23 11:41:34 -0400 | [diff] [blame] | 1855 | if (env.best_cpu == -1) |
| 1856 | nid = env.src_nid; |
| 1857 | else |
| 1858 | nid = env.dst_nid; |
| 1859 | |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1860 | if (ng->active_nodes > 1 && numa_is_active_node(env.dst_nid, ng)) |
Rik van Riel | db015da | 2014-06-23 11:41:34 -0400 | [diff] [blame] | 1861 | sched_setnuma(p, env.dst_nid); |
| 1862 | } |
| 1863 | |
| 1864 | /* No better CPU than the current one was found. */ |
| 1865 | if (env.best_cpu == -1) |
| 1866 | return -EAGAIN; |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 1867 | |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1868 | /* |
| 1869 | * Reset the scan period if the task is being rescheduled on an |
| 1870 | * alternative node to recheck if the tasks is now properly placed. |
| 1871 | */ |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 1872 | p->numa_scan_period = task_scan_start(p); |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1873 | |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1874 | if (env.best_task == NULL) { |
Mel Gorman | 286549d | 2014-01-21 15:51:03 -0800 | [diff] [blame] | 1875 | ret = migrate_task_to(p, env.best_cpu); |
| 1876 | if (ret != 0) |
| 1877 | trace_sched_stick_numa(p, env.src_cpu, env.best_cpu); |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1878 | return ret; |
| 1879 | } |
| 1880 | |
| 1881 | ret = migrate_swap(p, env.best_task); |
Mel Gorman | 286549d | 2014-01-21 15:51:03 -0800 | [diff] [blame] | 1882 | if (ret != 0) |
| 1883 | trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task)); |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 1884 | put_task_struct(env.best_task); |
| 1885 | return ret; |
Mel Gorman | e6628d5 | 2013-10-07 11:29:02 +0100 | [diff] [blame] | 1886 | } |
| 1887 | |
Mel Gorman | 6b9a746 | 2013-10-07 11:29:11 +0100 | [diff] [blame] | 1888 | /* Attempt to migrate a task to a CPU on the preferred node. */ |
| 1889 | static void numa_migrate_preferred(struct task_struct *p) |
| 1890 | { |
Rik van Riel | 5085e2a | 2014-04-11 13:00:28 -0400 | [diff] [blame] | 1891 | unsigned long interval = HZ; |
| 1892 | |
Rik van Riel | 2739d3e | 2013-10-07 11:29:41 +0100 | [diff] [blame] | 1893 | /* This task has no NUMA fault statistics yet */ |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 1894 | if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults)) |
Rik van Riel | 2739d3e | 2013-10-07 11:29:41 +0100 | [diff] [blame] | 1895 | return; |
| 1896 | |
| 1897 | /* Periodically retry migrating the task to the preferred node */ |
Rik van Riel | 5085e2a | 2014-04-11 13:00:28 -0400 | [diff] [blame] | 1898 | interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); |
| 1899 | p->numa_migrate_retry = jiffies + interval; |
Rik van Riel | 2739d3e | 2013-10-07 11:29:41 +0100 | [diff] [blame] | 1900 | |
Mel Gorman | 6b9a746 | 2013-10-07 11:29:11 +0100 | [diff] [blame] | 1901 | /* Success if task is already running on preferred CPU */ |
Wanpeng Li | de1b301 | 2013-12-12 15:23:24 +0800 | [diff] [blame] | 1902 | if (task_node(p) == p->numa_preferred_nid) |
Mel Gorman | 6b9a746 | 2013-10-07 11:29:11 +0100 | [diff] [blame] | 1903 | return; |
| 1904 | |
Mel Gorman | 6b9a746 | 2013-10-07 11:29:11 +0100 | [diff] [blame] | 1905 | /* Otherwise, try migrate to a CPU on the preferred node */ |
Rik van Riel | 2739d3e | 2013-10-07 11:29:41 +0100 | [diff] [blame] | 1906 | task_numa_migrate(p); |
Mel Gorman | 6b9a746 | 2013-10-07 11:29:11 +0100 | [diff] [blame] | 1907 | } |
| 1908 | |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1909 | /* |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1910 | * Find out how many nodes on the workload is actively running on. Do this by |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1911 | * tracking the nodes from which NUMA hinting faults are triggered. This can |
| 1912 | * be different from the set of nodes where the workload's memory is currently |
| 1913 | * located. |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1914 | */ |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1915 | static void numa_group_count_active_nodes(struct numa_group *numa_group) |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1916 | { |
| 1917 | unsigned long faults, max_faults = 0; |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1918 | int nid, active_nodes = 0; |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1919 | |
| 1920 | for_each_online_node(nid) { |
| 1921 | faults = group_faults_cpu(numa_group, nid); |
| 1922 | if (faults > max_faults) |
| 1923 | max_faults = faults; |
| 1924 | } |
| 1925 | |
| 1926 | for_each_online_node(nid) { |
| 1927 | faults = group_faults_cpu(numa_group, nid); |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1928 | if (faults * ACTIVE_NODE_FRACTION > max_faults) |
| 1929 | active_nodes++; |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1930 | } |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 1931 | |
| 1932 | numa_group->max_faults_cpu = max_faults; |
| 1933 | numa_group->active_nodes = active_nodes; |
Rik van Riel | 20e07de | 2014-01-27 17:03:43 -0500 | [diff] [blame] | 1934 | } |
| 1935 | |
| 1936 | /* |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1937 | * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS |
| 1938 | * increments. The more local the fault statistics are, the higher the scan |
Rik van Riel | a22b4b0 | 2014-06-23 11:41:35 -0400 | [diff] [blame] | 1939 | * period will be for the next scan window. If local/(local+remote) ratio is |
| 1940 | * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) |
| 1941 | * the scan period will decrease. Aim for 70% local accesses. |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1942 | */ |
| 1943 | #define NUMA_PERIOD_SLOTS 10 |
Rik van Riel | a22b4b0 | 2014-06-23 11:41:35 -0400 | [diff] [blame] | 1944 | #define NUMA_PERIOD_THRESHOLD 7 |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1945 | |
| 1946 | /* |
| 1947 | * Increase the scan period (slow down scanning) if the majority of |
| 1948 | * our memory is already on our local node, or if the majority of |
| 1949 | * the page accesses are shared with other processes. |
| 1950 | * Otherwise, decrease the scan period. |
| 1951 | */ |
| 1952 | static void update_task_scan_period(struct task_struct *p, |
| 1953 | unsigned long shared, unsigned long private) |
| 1954 | { |
| 1955 | unsigned int period_slot; |
Rik van Riel | 37ec97de | 2017-07-31 15:28:46 -0400 | [diff] [blame] | 1956 | int lr_ratio, ps_ratio; |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1957 | int diff; |
| 1958 | |
| 1959 | unsigned long remote = p->numa_faults_locality[0]; |
| 1960 | unsigned long local = p->numa_faults_locality[1]; |
| 1961 | |
| 1962 | /* |
| 1963 | * If there were no record hinting faults then either the task is |
| 1964 | * completely idle or all activity is areas that are not of interest |
Mel Gorman | 074c238 | 2015-03-25 15:55:42 -0700 | [diff] [blame] | 1965 | * to automatic numa balancing. Related to that, if there were failed |
| 1966 | * migration then it implies we are migrating too quickly or the local |
| 1967 | * node is overloaded. In either case, scan slower |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1968 | */ |
Mel Gorman | 074c238 | 2015-03-25 15:55:42 -0700 | [diff] [blame] | 1969 | if (local + shared == 0 || p->numa_faults_locality[2]) { |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 1970 | p->numa_scan_period = min(p->numa_scan_period_max, |
| 1971 | p->numa_scan_period << 1); |
| 1972 | |
| 1973 | p->mm->numa_next_scan = jiffies + |
| 1974 | msecs_to_jiffies(p->numa_scan_period); |
| 1975 | |
| 1976 | return; |
| 1977 | } |
| 1978 | |
| 1979 | /* |
| 1980 | * Prepare to scale scan period relative to the current period. |
| 1981 | * == NUMA_PERIOD_THRESHOLD scan period stays the same |
| 1982 | * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) |
| 1983 | * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) |
| 1984 | */ |
| 1985 | period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); |
Rik van Riel | 37ec97de | 2017-07-31 15:28:46 -0400 | [diff] [blame] | 1986 | lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); |
| 1987 | ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); |
| 1988 | |
| 1989 | if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { |
| 1990 | /* |
| 1991 | * Most memory accesses are local. There is no need to |
| 1992 | * do fast NUMA scanning, since memory is already local. |
| 1993 | */ |
| 1994 | int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; |
| 1995 | if (!slot) |
| 1996 | slot = 1; |
| 1997 | diff = slot * period_slot; |
| 1998 | } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { |
| 1999 | /* |
| 2000 | * Most memory accesses are shared with other tasks. |
| 2001 | * There is no point in continuing fast NUMA scanning, |
| 2002 | * since other tasks may just move the memory elsewhere. |
| 2003 | */ |
| 2004 | int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 2005 | if (!slot) |
| 2006 | slot = 1; |
| 2007 | diff = slot * period_slot; |
| 2008 | } else { |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 2009 | /* |
Rik van Riel | 37ec97de | 2017-07-31 15:28:46 -0400 | [diff] [blame] | 2010 | * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, |
| 2011 | * yet they are not on the local NUMA node. Speed up |
| 2012 | * NUMA scanning to get the memory moved over. |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 2013 | */ |
Rik van Riel | 37ec97de | 2017-07-31 15:28:46 -0400 | [diff] [blame] | 2014 | int ratio = max(lr_ratio, ps_ratio); |
| 2015 | diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 2016 | } |
| 2017 | |
| 2018 | p->numa_scan_period = clamp(p->numa_scan_period + diff, |
| 2019 | task_scan_min(p), task_scan_max(p)); |
| 2020 | memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); |
| 2021 | } |
| 2022 | |
Rik van Riel | 7e2703e | 2014-01-27 17:03:45 -0500 | [diff] [blame] | 2023 | /* |
| 2024 | * Get the fraction of time the task has been running since the last |
| 2025 | * NUMA placement cycle. The scheduler keeps similar statistics, but |
| 2026 | * decays those on a 32ms period, which is orders of magnitude off |
| 2027 | * from the dozens-of-seconds NUMA balancing period. Use the scheduler |
| 2028 | * stats only if the task is so new there are no NUMA statistics yet. |
| 2029 | */ |
| 2030 | static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) |
| 2031 | { |
| 2032 | u64 runtime, delta, now; |
| 2033 | /* Use the start of this time slice to avoid calculations. */ |
| 2034 | now = p->se.exec_start; |
| 2035 | runtime = p->se.sum_exec_runtime; |
| 2036 | |
| 2037 | if (p->last_task_numa_placement) { |
| 2038 | delta = runtime - p->last_sum_exec_runtime; |
| 2039 | *period = now - p->last_task_numa_placement; |
| 2040 | } else { |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 2041 | delta = p->se.avg.load_sum / p->se.load.weight; |
| 2042 | *period = LOAD_AVG_MAX; |
Rik van Riel | 7e2703e | 2014-01-27 17:03:45 -0500 | [diff] [blame] | 2043 | } |
| 2044 | |
| 2045 | p->last_sum_exec_runtime = runtime; |
| 2046 | p->last_task_numa_placement = now; |
| 2047 | |
| 2048 | return delta; |
| 2049 | } |
| 2050 | |
Rik van Riel | 5400941 | 2014-10-17 03:29:53 -0400 | [diff] [blame] | 2051 | /* |
| 2052 | * Determine the preferred nid for a task in a numa_group. This needs to |
| 2053 | * be done in a way that produces consistent results with group_weight, |
| 2054 | * otherwise workloads might not converge. |
| 2055 | */ |
| 2056 | static int preferred_group_nid(struct task_struct *p, int nid) |
| 2057 | { |
| 2058 | nodemask_t nodes; |
| 2059 | int dist; |
| 2060 | |
| 2061 | /* Direct connections between all NUMA nodes. */ |
| 2062 | if (sched_numa_topology_type == NUMA_DIRECT) |
| 2063 | return nid; |
| 2064 | |
| 2065 | /* |
| 2066 | * On a system with glueless mesh NUMA topology, group_weight |
| 2067 | * scores nodes according to the number of NUMA hinting faults on |
| 2068 | * both the node itself, and on nearby nodes. |
| 2069 | */ |
| 2070 | if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { |
| 2071 | unsigned long score, max_score = 0; |
| 2072 | int node, max_node = nid; |
| 2073 | |
| 2074 | dist = sched_max_numa_distance; |
| 2075 | |
| 2076 | for_each_online_node(node) { |
| 2077 | score = group_weight(p, node, dist); |
| 2078 | if (score > max_score) { |
| 2079 | max_score = score; |
| 2080 | max_node = node; |
| 2081 | } |
| 2082 | } |
| 2083 | return max_node; |
| 2084 | } |
| 2085 | |
| 2086 | /* |
| 2087 | * Finding the preferred nid in a system with NUMA backplane |
| 2088 | * interconnect topology is more involved. The goal is to locate |
| 2089 | * tasks from numa_groups near each other in the system, and |
| 2090 | * untangle workloads from different sides of the system. This requires |
| 2091 | * searching down the hierarchy of node groups, recursively searching |
| 2092 | * inside the highest scoring group of nodes. The nodemask tricks |
| 2093 | * keep the complexity of the search down. |
| 2094 | */ |
| 2095 | nodes = node_online_map; |
| 2096 | for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { |
| 2097 | unsigned long max_faults = 0; |
Jan Beulich | 8190747 | 2015-01-23 08:25:38 +0000 | [diff] [blame] | 2098 | nodemask_t max_group = NODE_MASK_NONE; |
Rik van Riel | 5400941 | 2014-10-17 03:29:53 -0400 | [diff] [blame] | 2099 | int a, b; |
| 2100 | |
| 2101 | /* Are there nodes at this distance from each other? */ |
| 2102 | if (!find_numa_distance(dist)) |
| 2103 | continue; |
| 2104 | |
| 2105 | for_each_node_mask(a, nodes) { |
| 2106 | unsigned long faults = 0; |
| 2107 | nodemask_t this_group; |
| 2108 | nodes_clear(this_group); |
| 2109 | |
| 2110 | /* Sum group's NUMA faults; includes a==b case. */ |
| 2111 | for_each_node_mask(b, nodes) { |
| 2112 | if (node_distance(a, b) < dist) { |
| 2113 | faults += group_faults(p, b); |
| 2114 | node_set(b, this_group); |
| 2115 | node_clear(b, nodes); |
| 2116 | } |
| 2117 | } |
| 2118 | |
| 2119 | /* Remember the top group. */ |
| 2120 | if (faults > max_faults) { |
| 2121 | max_faults = faults; |
| 2122 | max_group = this_group; |
| 2123 | /* |
| 2124 | * subtle: at the smallest distance there is |
| 2125 | * just one node left in each "group", the |
| 2126 | * winner is the preferred nid. |
| 2127 | */ |
| 2128 | nid = a; |
| 2129 | } |
| 2130 | } |
| 2131 | /* Next round, evaluate the nodes within max_group. */ |
Jan Beulich | 890a540 | 2015-02-09 12:30:00 +0100 | [diff] [blame] | 2132 | if (!max_faults) |
| 2133 | break; |
Rik van Riel | 5400941 | 2014-10-17 03:29:53 -0400 | [diff] [blame] | 2134 | nodes = max_group; |
| 2135 | } |
| 2136 | return nid; |
| 2137 | } |
| 2138 | |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2139 | static void task_numa_placement(struct task_struct *p) |
| 2140 | { |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 2141 | int seq, nid, max_nid = -1, max_group_nid = -1; |
| 2142 | unsigned long max_faults = 0, max_group_faults = 0; |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 2143 | unsigned long fault_types[2] = { 0, 0 }; |
Rik van Riel | 7e2703e | 2014-01-27 17:03:45 -0500 | [diff] [blame] | 2144 | unsigned long total_faults; |
| 2145 | u64 runtime, period; |
Mel Gorman | 7dbd13e | 2013-10-07 11:29:29 +0100 | [diff] [blame] | 2146 | spinlock_t *group_lock = NULL; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2147 | |
Jason Low | 7e5a2c1 | 2015-04-30 17:28:14 -0700 | [diff] [blame] | 2148 | /* |
| 2149 | * The p->mm->numa_scan_seq field gets updated without |
| 2150 | * exclusive access. Use READ_ONCE() here to ensure |
| 2151 | * that the field is read in a single access: |
| 2152 | */ |
Jason Low | 316c1608d | 2015-04-28 13:00:20 -0700 | [diff] [blame] | 2153 | seq = READ_ONCE(p->mm->numa_scan_seq); |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2154 | if (p->numa_scan_seq == seq) |
| 2155 | return; |
| 2156 | p->numa_scan_seq = seq; |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 2157 | p->numa_scan_period_max = task_scan_max(p); |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2158 | |
Rik van Riel | 7e2703e | 2014-01-27 17:03:45 -0500 | [diff] [blame] | 2159 | total_faults = p->numa_faults_locality[0] + |
| 2160 | p->numa_faults_locality[1]; |
| 2161 | runtime = numa_get_avg_runtime(p, &period); |
| 2162 | |
Mel Gorman | 7dbd13e | 2013-10-07 11:29:29 +0100 | [diff] [blame] | 2163 | /* If the task is part of a group prevent parallel updates to group stats */ |
| 2164 | if (p->numa_group) { |
| 2165 | group_lock = &p->numa_group->lock; |
Mike Galbraith | 60e69ee | 2014-04-07 10:55:15 +0200 | [diff] [blame] | 2166 | spin_lock_irq(group_lock); |
Mel Gorman | 7dbd13e | 2013-10-07 11:29:29 +0100 | [diff] [blame] | 2167 | } |
| 2168 | |
Mel Gorman | 688b758 | 2013-10-07 11:28:58 +0100 | [diff] [blame] | 2169 | /* Find the node with the highest number of faults */ |
| 2170 | for_each_online_node(nid) { |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2171 | /* Keep track of the offsets in numa_faults array */ |
| 2172 | int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 2173 | unsigned long faults = 0, group_faults = 0; |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2174 | int priv; |
Mel Gorman | 745d614 | 2013-10-07 11:28:59 +0100 | [diff] [blame] | 2175 | |
Rik van Riel | be1e4e7 | 2014-01-27 17:03:48 -0500 | [diff] [blame] | 2176 | for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { |
Rik van Riel | 7e2703e | 2014-01-27 17:03:45 -0500 | [diff] [blame] | 2177 | long diff, f_diff, f_weight; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2178 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2179 | mem_idx = task_faults_idx(NUMA_MEM, nid, priv); |
| 2180 | membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); |
| 2181 | cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); |
| 2182 | cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); |
Mel Gorman | 745d614 | 2013-10-07 11:28:59 +0100 | [diff] [blame] | 2183 | |
Mel Gorman | ac8e895 | 2013-10-07 11:29:03 +0100 | [diff] [blame] | 2184 | /* Decay existing window, copy faults since last scan */ |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2185 | diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; |
| 2186 | fault_types[priv] += p->numa_faults[membuf_idx]; |
| 2187 | p->numa_faults[membuf_idx] = 0; |
Mel Gorman | fb13c7e | 2013-10-07 11:29:17 +0100 | [diff] [blame] | 2188 | |
Rik van Riel | 7e2703e | 2014-01-27 17:03:45 -0500 | [diff] [blame] | 2189 | /* |
| 2190 | * Normalize the faults_from, so all tasks in a group |
| 2191 | * count according to CPU use, instead of by the raw |
| 2192 | * number of faults. Tasks with little runtime have |
| 2193 | * little over-all impact on throughput, and thus their |
| 2194 | * faults are less important. |
| 2195 | */ |
| 2196 | f_weight = div64_u64(runtime << 16, period + 1); |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2197 | f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / |
Rik van Riel | 7e2703e | 2014-01-27 17:03:45 -0500 | [diff] [blame] | 2198 | (total_faults + 1); |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2199 | f_diff = f_weight - p->numa_faults[cpu_idx] / 2; |
| 2200 | p->numa_faults[cpubuf_idx] = 0; |
Rik van Riel | 50ec8a4 | 2014-01-27 17:03:42 -0500 | [diff] [blame] | 2201 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2202 | p->numa_faults[mem_idx] += diff; |
| 2203 | p->numa_faults[cpu_idx] += f_diff; |
| 2204 | faults += p->numa_faults[mem_idx]; |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 2205 | p->total_numa_faults += diff; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2206 | if (p->numa_group) { |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2207 | /* |
| 2208 | * safe because we can only change our own group |
| 2209 | * |
| 2210 | * mem_idx represents the offset for a given |
| 2211 | * nid and priv in a specific region because it |
| 2212 | * is at the beginning of the numa_faults array. |
| 2213 | */ |
| 2214 | p->numa_group->faults[mem_idx] += diff; |
| 2215 | p->numa_group->faults_cpu[mem_idx] += f_diff; |
Mel Gorman | 989348b | 2013-10-07 11:29:40 +0100 | [diff] [blame] | 2216 | p->numa_group->total_faults += diff; |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2217 | group_faults += p->numa_group->faults[mem_idx]; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2218 | } |
Mel Gorman | ac8e895 | 2013-10-07 11:29:03 +0100 | [diff] [blame] | 2219 | } |
| 2220 | |
Mel Gorman | 688b758 | 2013-10-07 11:28:58 +0100 | [diff] [blame] | 2221 | if (faults > max_faults) { |
| 2222 | max_faults = faults; |
| 2223 | max_nid = nid; |
| 2224 | } |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 2225 | |
| 2226 | if (group_faults > max_group_faults) { |
| 2227 | max_group_faults = group_faults; |
| 2228 | max_group_nid = nid; |
| 2229 | } |
| 2230 | } |
| 2231 | |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 2232 | update_task_scan_period(p, fault_types[0], fault_types[1]); |
| 2233 | |
Mel Gorman | 7dbd13e | 2013-10-07 11:29:29 +0100 | [diff] [blame] | 2234 | if (p->numa_group) { |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 2235 | numa_group_count_active_nodes(p->numa_group); |
Mike Galbraith | 60e69ee | 2014-04-07 10:55:15 +0200 | [diff] [blame] | 2236 | spin_unlock_irq(group_lock); |
Rik van Riel | 5400941 | 2014-10-17 03:29:53 -0400 | [diff] [blame] | 2237 | max_nid = preferred_group_nid(p, max_group_nid); |
Mel Gorman | 688b758 | 2013-10-07 11:28:58 +0100 | [diff] [blame] | 2238 | } |
| 2239 | |
Rik van Riel | bb97fc3 | 2014-06-04 16:33:15 -0400 | [diff] [blame] | 2240 | if (max_faults) { |
| 2241 | /* Set the new preferred node */ |
| 2242 | if (max_nid != p->numa_preferred_nid) |
| 2243 | sched_setnuma(p, max_nid); |
| 2244 | |
| 2245 | if (task_node(p) != p->numa_preferred_nid) |
| 2246 | numa_migrate_preferred(p); |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 2247 | } |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2248 | } |
| 2249 | |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2250 | static inline int get_numa_group(struct numa_group *grp) |
| 2251 | { |
| 2252 | return atomic_inc_not_zero(&grp->refcount); |
| 2253 | } |
| 2254 | |
| 2255 | static inline void put_numa_group(struct numa_group *grp) |
| 2256 | { |
| 2257 | if (atomic_dec_and_test(&grp->refcount)) |
| 2258 | kfree_rcu(grp, rcu); |
| 2259 | } |
| 2260 | |
Mel Gorman | 3e6a941 | 2013-10-07 11:29:35 +0100 | [diff] [blame] | 2261 | static void task_numa_group(struct task_struct *p, int cpupid, int flags, |
| 2262 | int *priv) |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2263 | { |
| 2264 | struct numa_group *grp, *my_grp; |
| 2265 | struct task_struct *tsk; |
| 2266 | bool join = false; |
| 2267 | int cpu = cpupid_to_cpu(cpupid); |
| 2268 | int i; |
| 2269 | |
| 2270 | if (unlikely(!p->numa_group)) { |
| 2271 | unsigned int size = sizeof(struct numa_group) + |
Rik van Riel | 50ec8a4 | 2014-01-27 17:03:42 -0500 | [diff] [blame] | 2272 | 4*nr_node_ids*sizeof(unsigned long); |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2273 | |
| 2274 | grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); |
| 2275 | if (!grp) |
| 2276 | return; |
| 2277 | |
| 2278 | atomic_set(&grp->refcount, 1); |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 2279 | grp->active_nodes = 1; |
| 2280 | grp->max_faults_cpu = 0; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2281 | spin_lock_init(&grp->lock); |
Mel Gorman | e29cf08 | 2013-10-07 11:29:22 +0100 | [diff] [blame] | 2282 | grp->gid = p->pid; |
Rik van Riel | 50ec8a4 | 2014-01-27 17:03:42 -0500 | [diff] [blame] | 2283 | /* Second half of the array tracks nids where faults happen */ |
Rik van Riel | be1e4e7 | 2014-01-27 17:03:48 -0500 | [diff] [blame] | 2284 | grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * |
| 2285 | nr_node_ids; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2286 | |
Rik van Riel | be1e4e7 | 2014-01-27 17:03:48 -0500 | [diff] [blame] | 2287 | for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2288 | grp->faults[i] = p->numa_faults[i]; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2289 | |
Mel Gorman | 989348b | 2013-10-07 11:29:40 +0100 | [diff] [blame] | 2290 | grp->total_faults = p->total_numa_faults; |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 2291 | |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2292 | grp->nr_tasks++; |
| 2293 | rcu_assign_pointer(p->numa_group, grp); |
| 2294 | } |
| 2295 | |
| 2296 | rcu_read_lock(); |
Jason Low | 316c1608d | 2015-04-28 13:00:20 -0700 | [diff] [blame] | 2297 | tsk = READ_ONCE(cpu_rq(cpu)->curr); |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2298 | |
| 2299 | if (!cpupid_match_pid(tsk, cpupid)) |
Peter Zijlstra | 3354781 | 2013-10-09 10:24:48 +0200 | [diff] [blame] | 2300 | goto no_join; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2301 | |
| 2302 | grp = rcu_dereference(tsk->numa_group); |
| 2303 | if (!grp) |
Peter Zijlstra | 3354781 | 2013-10-09 10:24:48 +0200 | [diff] [blame] | 2304 | goto no_join; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2305 | |
| 2306 | my_grp = p->numa_group; |
| 2307 | if (grp == my_grp) |
Peter Zijlstra | 3354781 | 2013-10-09 10:24:48 +0200 | [diff] [blame] | 2308 | goto no_join; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2309 | |
| 2310 | /* |
| 2311 | * Only join the other group if its bigger; if we're the bigger group, |
| 2312 | * the other task will join us. |
| 2313 | */ |
| 2314 | if (my_grp->nr_tasks > grp->nr_tasks) |
Peter Zijlstra | 3354781 | 2013-10-09 10:24:48 +0200 | [diff] [blame] | 2315 | goto no_join; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2316 | |
| 2317 | /* |
| 2318 | * Tie-break on the grp address. |
| 2319 | */ |
| 2320 | if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) |
Peter Zijlstra | 3354781 | 2013-10-09 10:24:48 +0200 | [diff] [blame] | 2321 | goto no_join; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2322 | |
Rik van Riel | dabe1d9 | 2013-10-07 11:29:34 +0100 | [diff] [blame] | 2323 | /* Always join threads in the same process. */ |
| 2324 | if (tsk->mm == current->mm) |
| 2325 | join = true; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2326 | |
Rik van Riel | dabe1d9 | 2013-10-07 11:29:34 +0100 | [diff] [blame] | 2327 | /* Simple filter to avoid false positives due to PID collisions */ |
| 2328 | if (flags & TNF_SHARED) |
| 2329 | join = true; |
| 2330 | |
Mel Gorman | 3e6a941 | 2013-10-07 11:29:35 +0100 | [diff] [blame] | 2331 | /* Update priv based on whether false sharing was detected */ |
| 2332 | *priv = !join; |
| 2333 | |
Rik van Riel | dabe1d9 | 2013-10-07 11:29:34 +0100 | [diff] [blame] | 2334 | if (join && !get_numa_group(grp)) |
Peter Zijlstra | 3354781 | 2013-10-09 10:24:48 +0200 | [diff] [blame] | 2335 | goto no_join; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2336 | |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2337 | rcu_read_unlock(); |
| 2338 | |
| 2339 | if (!join) |
| 2340 | return; |
| 2341 | |
Mike Galbraith | 60e69ee | 2014-04-07 10:55:15 +0200 | [diff] [blame] | 2342 | BUG_ON(irqs_disabled()); |
| 2343 | double_lock_irq(&my_grp->lock, &grp->lock); |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2344 | |
Rik van Riel | be1e4e7 | 2014-01-27 17:03:48 -0500 | [diff] [blame] | 2345 | for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2346 | my_grp->faults[i] -= p->numa_faults[i]; |
| 2347 | grp->faults[i] += p->numa_faults[i]; |
Mel Gorman | 989348b | 2013-10-07 11:29:40 +0100 | [diff] [blame] | 2348 | } |
| 2349 | my_grp->total_faults -= p->total_numa_faults; |
| 2350 | grp->total_faults += p->total_numa_faults; |
| 2351 | |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2352 | my_grp->nr_tasks--; |
| 2353 | grp->nr_tasks++; |
| 2354 | |
| 2355 | spin_unlock(&my_grp->lock); |
Mike Galbraith | 60e69ee | 2014-04-07 10:55:15 +0200 | [diff] [blame] | 2356 | spin_unlock_irq(&grp->lock); |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2357 | |
| 2358 | rcu_assign_pointer(p->numa_group, grp); |
| 2359 | |
| 2360 | put_numa_group(my_grp); |
Peter Zijlstra | 3354781 | 2013-10-09 10:24:48 +0200 | [diff] [blame] | 2361 | return; |
| 2362 | |
| 2363 | no_join: |
| 2364 | rcu_read_unlock(); |
| 2365 | return; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2366 | } |
| 2367 | |
| 2368 | void task_numa_free(struct task_struct *p) |
| 2369 | { |
| 2370 | struct numa_group *grp = p->numa_group; |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2371 | void *numa_faults = p->numa_faults; |
Steven Rostedt | e9dd685 | 2014-05-27 17:02:04 -0400 | [diff] [blame] | 2372 | unsigned long flags; |
| 2373 | int i; |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2374 | |
| 2375 | if (grp) { |
Steven Rostedt | e9dd685 | 2014-05-27 17:02:04 -0400 | [diff] [blame] | 2376 | spin_lock_irqsave(&grp->lock, flags); |
Rik van Riel | be1e4e7 | 2014-01-27 17:03:48 -0500 | [diff] [blame] | 2377 | for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2378 | grp->faults[i] -= p->numa_faults[i]; |
Mel Gorman | 989348b | 2013-10-07 11:29:40 +0100 | [diff] [blame] | 2379 | grp->total_faults -= p->total_numa_faults; |
| 2380 | |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2381 | grp->nr_tasks--; |
Steven Rostedt | e9dd685 | 2014-05-27 17:02:04 -0400 | [diff] [blame] | 2382 | spin_unlock_irqrestore(&grp->lock, flags); |
Andreea-Cristina Bernat | 35b123e | 2014-08-22 17:50:43 +0300 | [diff] [blame] | 2383 | RCU_INIT_POINTER(p->numa_group, NULL); |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2384 | put_numa_group(grp); |
| 2385 | } |
| 2386 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2387 | p->numa_faults = NULL; |
Rik van Riel | 8272701 | 2013-10-07 11:29:28 +0100 | [diff] [blame] | 2388 | kfree(numa_faults); |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2389 | } |
| 2390 | |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2391 | /* |
| 2392 | * Got a PROT_NONE fault for a page on @node. |
| 2393 | */ |
Rik van Riel | 58b46da | 2014-01-27 17:03:47 -0500 | [diff] [blame] | 2394 | void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2395 | { |
| 2396 | struct task_struct *p = current; |
Peter Zijlstra | 6688cc0 | 2013-10-07 11:29:24 +0100 | [diff] [blame] | 2397 | bool migrated = flags & TNF_MIGRATED; |
Rik van Riel | 58b46da | 2014-01-27 17:03:47 -0500 | [diff] [blame] | 2398 | int cpu_node = task_node(current); |
Rik van Riel | 792568e | 2014-04-11 13:00:27 -0400 | [diff] [blame] | 2399 | int local = !!(flags & TNF_FAULT_LOCAL); |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 2400 | struct numa_group *ng; |
Mel Gorman | ac8e895 | 2013-10-07 11:29:03 +0100 | [diff] [blame] | 2401 | int priv; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2402 | |
Srikar Dronamraju | 2a59572 | 2015-08-11 21:54:21 +0530 | [diff] [blame] | 2403 | if (!static_branch_likely(&sched_numa_balancing)) |
Mel Gorman | 1a687c2 | 2012-11-22 11:16:36 +0000 | [diff] [blame] | 2404 | return; |
| 2405 | |
Mel Gorman | 9ff1d9f | 2013-10-07 11:29:04 +0100 | [diff] [blame] | 2406 | /* for example, ksmd faulting in a user's mm */ |
| 2407 | if (!p->mm) |
| 2408 | return; |
| 2409 | |
Mel Gorman | f809ca9 | 2013-10-07 11:28:57 +0100 | [diff] [blame] | 2410 | /* Allocate buffer to track faults on a per-node basis */ |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2411 | if (unlikely(!p->numa_faults)) { |
| 2412 | int size = sizeof(*p->numa_faults) * |
Rik van Riel | be1e4e7 | 2014-01-27 17:03:48 -0500 | [diff] [blame] | 2413 | NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; |
Mel Gorman | f809ca9 | 2013-10-07 11:28:57 +0100 | [diff] [blame] | 2414 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2415 | p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); |
| 2416 | if (!p->numa_faults) |
Mel Gorman | f809ca9 | 2013-10-07 11:28:57 +0100 | [diff] [blame] | 2417 | return; |
Mel Gorman | 745d614 | 2013-10-07 11:28:59 +0100 | [diff] [blame] | 2418 | |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 2419 | p->total_numa_faults = 0; |
Rik van Riel | 04bb2f9 | 2013-10-07 11:29:36 +0100 | [diff] [blame] | 2420 | memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); |
Mel Gorman | f809ca9 | 2013-10-07 11:28:57 +0100 | [diff] [blame] | 2421 | } |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2422 | |
Mel Gorman | fb003b8 | 2012-11-15 09:01:14 +0000 | [diff] [blame] | 2423 | /* |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2424 | * First accesses are treated as private, otherwise consider accesses |
| 2425 | * to be private if the accessing pid has not changed |
| 2426 | */ |
| 2427 | if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { |
| 2428 | priv = 1; |
| 2429 | } else { |
| 2430 | priv = cpupid_match_pid(p, last_cpupid); |
Peter Zijlstra | 6688cc0 | 2013-10-07 11:29:24 +0100 | [diff] [blame] | 2431 | if (!priv && !(flags & TNF_NO_GROUP)) |
Mel Gorman | 3e6a941 | 2013-10-07 11:29:35 +0100 | [diff] [blame] | 2432 | task_numa_group(p, last_cpupid, flags, &priv); |
Peter Zijlstra | 8c8a743 | 2013-10-07 11:29:21 +0100 | [diff] [blame] | 2433 | } |
| 2434 | |
Rik van Riel | 792568e | 2014-04-11 13:00:27 -0400 | [diff] [blame] | 2435 | /* |
| 2436 | * If a workload spans multiple NUMA nodes, a shared fault that |
| 2437 | * occurs wholly within the set of nodes that the workload is |
| 2438 | * actively using should be counted as local. This allows the |
| 2439 | * scan rate to slow down when a workload has settled down. |
| 2440 | */ |
Rik van Riel | 4142c3e | 2016-01-25 17:07:39 -0500 | [diff] [blame] | 2441 | ng = p->numa_group; |
| 2442 | if (!priv && !local && ng && ng->active_nodes > 1 && |
| 2443 | numa_is_active_node(cpu_node, ng) && |
| 2444 | numa_is_active_node(mem_node, ng)) |
Rik van Riel | 792568e | 2014-04-11 13:00:27 -0400 | [diff] [blame] | 2445 | local = 1; |
| 2446 | |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2447 | task_numa_placement(p); |
Mel Gorman | f809ca9 | 2013-10-07 11:28:57 +0100 | [diff] [blame] | 2448 | |
Rik van Riel | 2739d3e | 2013-10-07 11:29:41 +0100 | [diff] [blame] | 2449 | /* |
| 2450 | * Retry task to preferred node migration periodically, in case it |
| 2451 | * case it previously failed, or the scheduler moved us. |
| 2452 | */ |
| 2453 | if (time_after(jiffies, p->numa_migrate_retry)) |
Mel Gorman | 6b9a746 | 2013-10-07 11:29:11 +0100 | [diff] [blame] | 2454 | numa_migrate_preferred(p); |
| 2455 | |
Ingo Molnar | b32e86b | 2013-10-07 11:29:30 +0100 | [diff] [blame] | 2456 | if (migrated) |
| 2457 | p->numa_pages_migrated += pages; |
Mel Gorman | 074c238 | 2015-03-25 15:55:42 -0700 | [diff] [blame] | 2458 | if (flags & TNF_MIGRATE_FAIL) |
| 2459 | p->numa_faults_locality[2] += pages; |
Ingo Molnar | b32e86b | 2013-10-07 11:29:30 +0100 | [diff] [blame] | 2460 | |
Iulia Manda | 44dba3d | 2014-10-31 02:13:31 +0200 | [diff] [blame] | 2461 | p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; |
| 2462 | p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; |
Rik van Riel | 792568e | 2014-04-11 13:00:27 -0400 | [diff] [blame] | 2463 | p->numa_faults_locality[local] += pages; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2464 | } |
| 2465 | |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2466 | static void reset_ptenuma_scan(struct task_struct *p) |
| 2467 | { |
Jason Low | 7e5a2c1 | 2015-04-30 17:28:14 -0700 | [diff] [blame] | 2468 | /* |
| 2469 | * We only did a read acquisition of the mmap sem, so |
| 2470 | * p->mm->numa_scan_seq is written to without exclusive access |
| 2471 | * and the update is not guaranteed to be atomic. That's not |
| 2472 | * much of an issue though, since this is just used for |
| 2473 | * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not |
| 2474 | * expensive, to avoid any form of compiler optimizations: |
| 2475 | */ |
Jason Low | 316c1608d | 2015-04-28 13:00:20 -0700 | [diff] [blame] | 2476 | WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2477 | p->mm->numa_scan_offset = 0; |
| 2478 | } |
| 2479 | |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2480 | /* |
| 2481 | * The expensive part of numa migration is done from task_work context. |
| 2482 | * Triggered from task_tick_numa(). |
| 2483 | */ |
| 2484 | void task_numa_work(struct callback_head *work) |
| 2485 | { |
| 2486 | unsigned long migrate, next_scan, now = jiffies; |
| 2487 | struct task_struct *p = current; |
| 2488 | struct mm_struct *mm = p->mm; |
Rik van Riel | 5117084 | 2015-11-05 15:56:23 -0500 | [diff] [blame] | 2489 | u64 runtime = p->se.sum_exec_runtime; |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2490 | struct vm_area_struct *vma; |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2491 | unsigned long start, end; |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 2492 | unsigned long nr_pte_updates = 0; |
Rik van Riel | 4620f8c | 2015-09-11 09:00:27 -0400 | [diff] [blame] | 2493 | long pages, virtpages; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2494 | |
Peter Zijlstra | 9148a3a | 2016-09-20 22:34:51 +0200 | [diff] [blame] | 2495 | SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2496 | |
| 2497 | work->next = work; /* protect against double add */ |
| 2498 | /* |
| 2499 | * Who cares about NUMA placement when they're dying. |
| 2500 | * |
| 2501 | * NOTE: make sure not to dereference p->mm before this check, |
| 2502 | * exit_task_work() happens _after_ exit_mm() so we could be called |
| 2503 | * without p->mm even though we still had it when we enqueued this |
| 2504 | * work. |
| 2505 | */ |
| 2506 | if (p->flags & PF_EXITING) |
| 2507 | return; |
| 2508 | |
Mel Gorman | 930aa17 | 2013-10-07 11:29:37 +0100 | [diff] [blame] | 2509 | if (!mm->numa_next_scan) { |
Mel Gorman | 7e8d16b | 2013-10-07 11:28:54 +0100 | [diff] [blame] | 2510 | mm->numa_next_scan = now + |
| 2511 | msecs_to_jiffies(sysctl_numa_balancing_scan_delay); |
Mel Gorman | b8593bf | 2012-11-21 01:18:23 +0000 | [diff] [blame] | 2512 | } |
| 2513 | |
| 2514 | /* |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2515 | * Enforce maximal scan/migration frequency.. |
| 2516 | */ |
| 2517 | migrate = mm->numa_next_scan; |
| 2518 | if (time_before(now, migrate)) |
| 2519 | return; |
| 2520 | |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 2521 | if (p->numa_scan_period == 0) { |
| 2522 | p->numa_scan_period_max = task_scan_max(p); |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 2523 | p->numa_scan_period = task_scan_start(p); |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 2524 | } |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2525 | |
Mel Gorman | fb003b8 | 2012-11-15 09:01:14 +0000 | [diff] [blame] | 2526 | next_scan = now + msecs_to_jiffies(p->numa_scan_period); |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2527 | if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) |
| 2528 | return; |
| 2529 | |
Mel Gorman | e14808b | 2012-11-19 10:59:15 +0000 | [diff] [blame] | 2530 | /* |
Peter Zijlstra | 19a78d1 | 2013-10-07 11:28:51 +0100 | [diff] [blame] | 2531 | * Delay this task enough that another task of this mm will likely win |
| 2532 | * the next time around. |
| 2533 | */ |
| 2534 | p->node_stamp += 2 * TICK_NSEC; |
| 2535 | |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2536 | start = mm->numa_scan_offset; |
| 2537 | pages = sysctl_numa_balancing_scan_size; |
| 2538 | pages <<= 20 - PAGE_SHIFT; /* MB in pages */ |
Rik van Riel | 4620f8c | 2015-09-11 09:00:27 -0400 | [diff] [blame] | 2539 | virtpages = pages * 8; /* Scan up to this much virtual space */ |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2540 | if (!pages) |
| 2541 | return; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2542 | |
Rik van Riel | 4620f8c | 2015-09-11 09:00:27 -0400 | [diff] [blame] | 2543 | |
Vlastimil Babka | 8655d54 | 2017-05-15 15:13:16 +0200 | [diff] [blame] | 2544 | if (!down_read_trylock(&mm->mmap_sem)) |
| 2545 | return; |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2546 | vma = find_vma(mm, start); |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2547 | if (!vma) { |
| 2548 | reset_ptenuma_scan(p); |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2549 | start = 0; |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2550 | vma = mm->mmap; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2551 | } |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2552 | for (; vma; vma = vma->vm_next) { |
Naoya Horiguchi | 6b79c57 | 2015-04-07 14:26:47 -0700 | [diff] [blame] | 2553 | if (!vma_migratable(vma) || !vma_policy_mof(vma) || |
Mel Gorman | 8e76d4e | 2015-06-10 11:15:00 -0700 | [diff] [blame] | 2554 | is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2555 | continue; |
Naoya Horiguchi | 6b79c57 | 2015-04-07 14:26:47 -0700 | [diff] [blame] | 2556 | } |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2557 | |
Mel Gorman | 4591ce4f | 2013-10-07 11:29:13 +0100 | [diff] [blame] | 2558 | /* |
| 2559 | * Shared library pages mapped by multiple processes are not |
| 2560 | * migrated as it is expected they are cache replicated. Avoid |
| 2561 | * hinting faults in read-only file-backed mappings or the vdso |
| 2562 | * as migrating the pages will be of marginal benefit. |
| 2563 | */ |
| 2564 | if (!vma->vm_mm || |
| 2565 | (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) |
| 2566 | continue; |
| 2567 | |
Mel Gorman | 3c67f47 | 2013-12-18 17:08:40 -0800 | [diff] [blame] | 2568 | /* |
| 2569 | * Skip inaccessible VMAs to avoid any confusion between |
| 2570 | * PROT_NONE and NUMA hinting ptes |
| 2571 | */ |
| 2572 | if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) |
| 2573 | continue; |
| 2574 | |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2575 | do { |
| 2576 | start = max(start, vma->vm_start); |
| 2577 | end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); |
| 2578 | end = min(end, vma->vm_end); |
Rik van Riel | 4620f8c | 2015-09-11 09:00:27 -0400 | [diff] [blame] | 2579 | nr_pte_updates = change_prot_numa(vma, start, end); |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 2580 | |
| 2581 | /* |
Rik van Riel | 4620f8c | 2015-09-11 09:00:27 -0400 | [diff] [blame] | 2582 | * Try to scan sysctl_numa_balancing_size worth of |
| 2583 | * hpages that have at least one present PTE that |
| 2584 | * is not already pte-numa. If the VMA contains |
| 2585 | * areas that are unused or already full of prot_numa |
| 2586 | * PTEs, scan up to virtpages, to skip through those |
| 2587 | * areas faster. |
Mel Gorman | 598f0ec | 2013-10-07 11:28:55 +0100 | [diff] [blame] | 2588 | */ |
| 2589 | if (nr_pte_updates) |
| 2590 | pages -= (end - start) >> PAGE_SHIFT; |
Rik van Riel | 4620f8c | 2015-09-11 09:00:27 -0400 | [diff] [blame] | 2591 | virtpages -= (end - start) >> PAGE_SHIFT; |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2592 | |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2593 | start = end; |
Rik van Riel | 4620f8c | 2015-09-11 09:00:27 -0400 | [diff] [blame] | 2594 | if (pages <= 0 || virtpages <= 0) |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2595 | goto out; |
Rik van Riel | 3cf1962 | 2014-02-18 17:12:44 -0500 | [diff] [blame] | 2596 | |
| 2597 | cond_resched(); |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2598 | } while (end != vma->vm_end); |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2599 | } |
| 2600 | |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2601 | out: |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2602 | /* |
Peter Zijlstra | c69307d | 2013-10-07 11:28:41 +0100 | [diff] [blame] | 2603 | * It is possible to reach the end of the VMA list but the last few |
| 2604 | * VMAs are not guaranteed to the vma_migratable. If they are not, we |
| 2605 | * would find the !migratable VMA on the next scan but not reset the |
| 2606 | * scanner to the start so check it now. |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2607 | */ |
| 2608 | if (vma) |
Mel Gorman | 9f40604 | 2012-11-14 18:34:32 +0000 | [diff] [blame] | 2609 | mm->numa_scan_offset = start; |
Peter Zijlstra | 6e5fb22 | 2012-10-25 14:16:45 +0200 | [diff] [blame] | 2610 | else |
| 2611 | reset_ptenuma_scan(p); |
| 2612 | up_read(&mm->mmap_sem); |
Rik van Riel | 5117084 | 2015-11-05 15:56:23 -0500 | [diff] [blame] | 2613 | |
| 2614 | /* |
| 2615 | * Make sure tasks use at least 32x as much time to run other code |
| 2616 | * than they used here, to limit NUMA PTE scanning overhead to 3% max. |
| 2617 | * Usually update_task_scan_period slows down scanning enough; on an |
| 2618 | * overloaded system we need to limit overhead on a per task basis. |
| 2619 | */ |
| 2620 | if (unlikely(p->se.sum_exec_runtime != runtime)) { |
| 2621 | u64 diff = p->se.sum_exec_runtime - runtime; |
| 2622 | p->node_stamp += 32 * diff; |
| 2623 | } |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2624 | } |
| 2625 | |
| 2626 | /* |
| 2627 | * Drive the periodic memory faults.. |
| 2628 | */ |
| 2629 | void task_tick_numa(struct rq *rq, struct task_struct *curr) |
| 2630 | { |
| 2631 | struct callback_head *work = &curr->numa_work; |
| 2632 | u64 period, now; |
| 2633 | |
| 2634 | /* |
| 2635 | * We don't care about NUMA placement if we don't have memory. |
| 2636 | */ |
| 2637 | if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) |
| 2638 | return; |
| 2639 | |
| 2640 | /* |
| 2641 | * Using runtime rather than walltime has the dual advantage that |
| 2642 | * we (mostly) drive the selection from busy threads and that the |
| 2643 | * task needs to have done some actual work before we bother with |
| 2644 | * NUMA placement. |
| 2645 | */ |
| 2646 | now = curr->se.sum_exec_runtime; |
| 2647 | period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; |
| 2648 | |
Rik van Riel | 25b3e5a | 2015-11-05 15:56:22 -0500 | [diff] [blame] | 2649 | if (now > curr->node_stamp + period) { |
Peter Zijlstra | 4b96a29 | 2012-10-25 14:16:47 +0200 | [diff] [blame] | 2650 | if (!curr->node_stamp) |
Rik van Riel | b5dd77c | 2017-07-31 15:28:47 -0400 | [diff] [blame] | 2651 | curr->numa_scan_period = task_scan_start(curr); |
Peter Zijlstra | 19a78d1 | 2013-10-07 11:28:51 +0100 | [diff] [blame] | 2652 | curr->node_stamp += period; |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2653 | |
| 2654 | if (!time_before(jiffies, curr->mm->numa_next_scan)) { |
| 2655 | init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ |
| 2656 | task_work_add(curr, work, true); |
| 2657 | } |
| 2658 | } |
| 2659 | } |
Rik van Riel | 3fed382 | 2017-06-23 12:55:29 -0400 | [diff] [blame] | 2660 | |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2661 | #else |
| 2662 | static void task_tick_numa(struct rq *rq, struct task_struct *curr) |
| 2663 | { |
| 2664 | } |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 2665 | |
| 2666 | static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) |
| 2667 | { |
| 2668 | } |
| 2669 | |
| 2670 | static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) |
| 2671 | { |
| 2672 | } |
Rik van Riel | 3fed382 | 2017-06-23 12:55:29 -0400 | [diff] [blame] | 2673 | |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 2674 | #endif /* CONFIG_NUMA_BALANCING */ |
| 2675 | |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 2676 | static void |
| 2677 | account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 2678 | { |
| 2679 | update_load_add(&cfs_rq->load, se->load.weight); |
Peter Zijlstra | c09595f | 2008-06-27 13:41:14 +0200 | [diff] [blame] | 2680 | if (!parent_entity(se)) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 2681 | update_load_add(&rq_of(cfs_rq)->load, se->load.weight); |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 2682 | #ifdef CONFIG_SMP |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 2683 | if (entity_is_task(se)) { |
| 2684 | struct rq *rq = rq_of(cfs_rq); |
| 2685 | |
| 2686 | account_numa_enqueue(rq, task_of(se)); |
| 2687 | list_add(&se->group_node, &rq->cfs_tasks); |
| 2688 | } |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 2689 | #endif |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 2690 | cfs_rq->nr_running++; |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 2691 | } |
| 2692 | |
| 2693 | static void |
| 2694 | account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 2695 | { |
| 2696 | update_load_sub(&cfs_rq->load, se->load.weight); |
Peter Zijlstra | c09595f | 2008-06-27 13:41:14 +0200 | [diff] [blame] | 2697 | if (!parent_entity(se)) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 2698 | update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); |
Tim Chen | bfdb198 | 2016-02-01 14:47:59 -0800 | [diff] [blame] | 2699 | #ifdef CONFIG_SMP |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 2700 | if (entity_is_task(se)) { |
| 2701 | account_numa_dequeue(rq_of(cfs_rq), task_of(se)); |
Bharata B Rao | b87f172 | 2008-09-25 09:53:54 +0530 | [diff] [blame] | 2702 | list_del_init(&se->group_node); |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 2703 | } |
Tim Chen | bfdb198 | 2016-02-01 14:47:59 -0800 | [diff] [blame] | 2704 | #endif |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 2705 | cfs_rq->nr_running--; |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 2706 | } |
| 2707 | |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2708 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 2709 | # ifdef CONFIG_SMP |
Paul Turner | 6d5ab29 | 2011-01-21 20:45:01 -0800 | [diff] [blame] | 2710 | static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2711 | { |
Peter Zijlstra | cf5f0ac | 2011-10-13 16:52:28 +0200 | [diff] [blame] | 2712 | long tg_weight, load, shares; |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2713 | |
Peter Zijlstra | ea1dc6f | 2016-06-24 16:11:02 +0200 | [diff] [blame] | 2714 | /* |
| 2715 | * This really should be: cfs_rq->avg.load_avg, but instead we use |
| 2716 | * cfs_rq->load.weight, which is its upper bound. This helps ramp up |
| 2717 | * the shares for small weight interactive tasks. |
| 2718 | */ |
| 2719 | load = scale_load_down(cfs_rq->load.weight); |
| 2720 | |
| 2721 | tg_weight = atomic_long_read(&tg->load_avg); |
| 2722 | |
| 2723 | /* Ensure tg_weight >= load */ |
| 2724 | tg_weight -= cfs_rq->tg_load_avg_contrib; |
| 2725 | tg_weight += load; |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2726 | |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2727 | shares = (tg->shares * load); |
Peter Zijlstra | cf5f0ac | 2011-10-13 16:52:28 +0200 | [diff] [blame] | 2728 | if (tg_weight) |
| 2729 | shares /= tg_weight; |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2730 | |
Dietmar Eggemann | b8fd842 | 2017-01-11 11:29:47 +0000 | [diff] [blame] | 2731 | /* |
| 2732 | * MIN_SHARES has to be unscaled here to support per-CPU partitioning |
| 2733 | * of a group with small tg->shares value. It is a floor value which is |
| 2734 | * assigned as a minimum load.weight to the sched_entity representing |
| 2735 | * the group on a CPU. |
| 2736 | * |
| 2737 | * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 |
| 2738 | * on an 8-core system with 8 tasks each runnable on one CPU shares has |
| 2739 | * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In |
| 2740 | * case no task is runnable on a CPU MIN_SHARES=2 should be returned |
| 2741 | * instead of 0. |
| 2742 | */ |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2743 | if (shares < MIN_SHARES) |
| 2744 | shares = MIN_SHARES; |
| 2745 | if (shares > tg->shares) |
| 2746 | shares = tg->shares; |
| 2747 | |
| 2748 | return shares; |
| 2749 | } |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2750 | # else /* CONFIG_SMP */ |
Paul Turner | 6d5ab29 | 2011-01-21 20:45:01 -0800 | [diff] [blame] | 2751 | static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2752 | { |
| 2753 | return tg->shares; |
| 2754 | } |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2755 | # endif /* CONFIG_SMP */ |
Peter Zijlstra | ea1dc6f | 2016-06-24 16:11:02 +0200 | [diff] [blame] | 2756 | |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2757 | static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, |
| 2758 | unsigned long weight) |
| 2759 | { |
Paul Turner | 19e5eeb | 2010-12-15 19:10:18 -0800 | [diff] [blame] | 2760 | if (se->on_rq) { |
| 2761 | /* commit outstanding execution time */ |
| 2762 | if (cfs_rq->curr == se) |
| 2763 | update_curr(cfs_rq); |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2764 | account_entity_dequeue(cfs_rq, se); |
Paul Turner | 19e5eeb | 2010-12-15 19:10:18 -0800 | [diff] [blame] | 2765 | } |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2766 | |
| 2767 | update_load_set(&se->load, weight); |
| 2768 | |
| 2769 | if (se->on_rq) |
| 2770 | account_entity_enqueue(cfs_rq, se); |
| 2771 | } |
| 2772 | |
Paul Turner | 8295836 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 2773 | static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); |
| 2774 | |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 2775 | static void update_cfs_shares(struct sched_entity *se) |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2776 | { |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 2777 | struct cfs_rq *cfs_rq = group_cfs_rq(se); |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2778 | struct task_group *tg; |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2779 | long shares; |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2780 | |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 2781 | if (!cfs_rq) |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2782 | return; |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 2783 | |
| 2784 | if (throttled_hierarchy(cfs_rq)) |
| 2785 | return; |
| 2786 | |
| 2787 | tg = cfs_rq->tg; |
| 2788 | |
Yong Zhang | 3ff6dca | 2011-01-24 15:33:52 +0800 | [diff] [blame] | 2789 | #ifndef CONFIG_SMP |
| 2790 | if (likely(se->load.weight == tg->shares)) |
| 2791 | return; |
| 2792 | #endif |
Paul Turner | 6d5ab29 | 2011-01-21 20:45:01 -0800 | [diff] [blame] | 2793 | shares = calc_cfs_shares(cfs_rq, tg); |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2794 | |
| 2795 | reweight_entity(cfs_rq_of(se), se, shares); |
| 2796 | } |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 2797 | |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2798 | #else /* CONFIG_FAIR_GROUP_SCHED */ |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 2799 | static inline void update_cfs_shares(struct sched_entity *se) |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 2800 | { |
| 2801 | } |
| 2802 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
| 2803 | |
Viresh Kumar | a030d73 | 2017-05-24 10:59:52 +0530 | [diff] [blame] | 2804 | static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq) |
| 2805 | { |
| 2806 | if (&this_rq()->cfs == cfs_rq) { |
| 2807 | /* |
| 2808 | * There are a few boundary cases this might miss but it should |
| 2809 | * get called often enough that that should (hopefully) not be |
| 2810 | * a real problem -- added to that it only calls on the local |
| 2811 | * CPU, so if we enqueue remotely we'll miss an update, but |
| 2812 | * the next tick/schedule should update. |
| 2813 | * |
| 2814 | * It will not get called when we go idle, because the idle |
| 2815 | * thread is a different class (!fair), nor will the utilization |
| 2816 | * number include things like RT tasks. |
| 2817 | * |
| 2818 | * As is, the util number is not freq-invariant (we'd have to |
| 2819 | * implement arch_scale_freq_capacity() for that). |
| 2820 | * |
| 2821 | * See cpu_util(). |
| 2822 | */ |
| 2823 | cpufreq_update_util(rq_of(cfs_rq), 0); |
| 2824 | } |
| 2825 | } |
| 2826 | |
Alex Shi | 141965c | 2013-06-26 13:05:39 +0800 | [diff] [blame] | 2827 | #ifdef CONFIG_SMP |
Paul Turner | 5b51f2f | 2012-10-04 13:18:32 +0200 | [diff] [blame] | 2828 | /* |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2829 | * Approximate: |
| 2830 | * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) |
| 2831 | */ |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2832 | static u64 decay_load(u64 val, u64 n) |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2833 | { |
Paul Turner | 5b51f2f | 2012-10-04 13:18:32 +0200 | [diff] [blame] | 2834 | unsigned int local_n; |
| 2835 | |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2836 | if (unlikely(n > LOAD_AVG_PERIOD * 63)) |
Paul Turner | 5b51f2f | 2012-10-04 13:18:32 +0200 | [diff] [blame] | 2837 | return 0; |
| 2838 | |
| 2839 | /* after bounds checking we can collapse to 32-bit */ |
| 2840 | local_n = n; |
| 2841 | |
| 2842 | /* |
| 2843 | * As y^PERIOD = 1/2, we can combine |
Zhihui Zhang | 9c58c79 | 2014-09-20 21:24:36 -0400 | [diff] [blame] | 2844 | * y^n = 1/2^(n/PERIOD) * y^(n%PERIOD) |
| 2845 | * With a look-up table which covers y^n (n<PERIOD) |
Paul Turner | 5b51f2f | 2012-10-04 13:18:32 +0200 | [diff] [blame] | 2846 | * |
| 2847 | * To achieve constant time decay_load. |
| 2848 | */ |
| 2849 | if (unlikely(local_n >= LOAD_AVG_PERIOD)) { |
| 2850 | val >>= local_n / LOAD_AVG_PERIOD; |
| 2851 | local_n %= LOAD_AVG_PERIOD; |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2852 | } |
| 2853 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 2854 | val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); |
| 2855 | return val; |
Paul Turner | 5b51f2f | 2012-10-04 13:18:32 +0200 | [diff] [blame] | 2856 | } |
| 2857 | |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2858 | static u32 __accumulate_pelt_segments(u64 periods, u32 d1, u32 d3) |
Paul Turner | 5b51f2f | 2012-10-04 13:18:32 +0200 | [diff] [blame] | 2859 | { |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2860 | u32 c1, c2, c3 = d3; /* y^0 == 1 */ |
Paul Turner | 5b51f2f | 2012-10-04 13:18:32 +0200 | [diff] [blame] | 2861 | |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2862 | /* |
Peter Zijlstra | 3841cdc | 2017-04-10 12:47:33 +0200 | [diff] [blame] | 2863 | * c1 = d1 y^p |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2864 | */ |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2865 | c1 = decay_load((u64)d1, periods); |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2866 | |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2867 | /* |
Peter Zijlstra | 3841cdc | 2017-04-10 12:47:33 +0200 | [diff] [blame] | 2868 | * p-1 |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2869 | * c2 = 1024 \Sum y^n |
| 2870 | * n=1 |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2871 | * |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2872 | * inf inf |
| 2873 | * = 1024 ( \Sum y^n - \Sum y^n - y^0 ) |
Peter Zijlstra | 3841cdc | 2017-04-10 12:47:33 +0200 | [diff] [blame] | 2874 | * n=0 n=p |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2875 | */ |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2876 | c2 = LOAD_AVG_MAX - decay_load(LOAD_AVG_MAX, periods) - 1024; |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2877 | |
| 2878 | return c1 + c2 + c3; |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2879 | } |
| 2880 | |
Peter Zijlstra | 54a2138 | 2015-09-07 15:05:42 +0200 | [diff] [blame] | 2881 | #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) |
Dietmar Eggemann | e0f5f3a | 2015-08-14 17:23:09 +0100 | [diff] [blame] | 2882 | |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2883 | /* |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2884 | * Accumulate the three separate parts of the sum; d1 the remainder |
| 2885 | * of the last (incomplete) period, d2 the span of full periods and d3 |
| 2886 | * the remainder of the (incomplete) current period. |
| 2887 | * |
| 2888 | * d1 d2 d3 |
| 2889 | * ^ ^ ^ |
| 2890 | * | | | |
| 2891 | * |<->|<----------------->|<--->| |
| 2892 | * ... |---x---|------| ... |------|-----x (now) |
| 2893 | * |
Peter Zijlstra | 3841cdc | 2017-04-10 12:47:33 +0200 | [diff] [blame] | 2894 | * p-1 |
| 2895 | * u' = (u + d1) y^p + 1024 \Sum y^n + d3 y^0 |
| 2896 | * n=1 |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2897 | * |
Peter Zijlstra | 3841cdc | 2017-04-10 12:47:33 +0200 | [diff] [blame] | 2898 | * = u y^p + (Step 1) |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2899 | * |
Peter Zijlstra | 3841cdc | 2017-04-10 12:47:33 +0200 | [diff] [blame] | 2900 | * p-1 |
| 2901 | * d1 y^p + 1024 \Sum y^n + d3 y^0 (Step 2) |
| 2902 | * n=1 |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2903 | */ |
| 2904 | static __always_inline u32 |
| 2905 | accumulate_sum(u64 delta, int cpu, struct sched_avg *sa, |
| 2906 | unsigned long weight, int running, struct cfs_rq *cfs_rq) |
| 2907 | { |
| 2908 | unsigned long scale_freq, scale_cpu; |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2909 | u32 contrib = (u32)delta; /* p == 0 -> delta < 1024 */ |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2910 | u64 periods; |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2911 | |
| 2912 | scale_freq = arch_scale_freq_capacity(NULL, cpu); |
| 2913 | scale_cpu = arch_scale_cpu_capacity(NULL, cpu); |
| 2914 | |
| 2915 | delta += sa->period_contrib; |
| 2916 | periods = delta / 1024; /* A period is 1024us (~1ms) */ |
| 2917 | |
| 2918 | /* |
| 2919 | * Step 1: decay old *_sum if we crossed period boundaries. |
| 2920 | */ |
| 2921 | if (periods) { |
| 2922 | sa->load_sum = decay_load(sa->load_sum, periods); |
| 2923 | if (cfs_rq) { |
| 2924 | cfs_rq->runnable_load_sum = |
| 2925 | decay_load(cfs_rq->runnable_load_sum, periods); |
| 2926 | } |
| 2927 | sa->util_sum = decay_load((u64)(sa->util_sum), periods); |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2928 | |
Peter Zijlstra | 05296e7 | 2017-03-31 10:51:41 +0200 | [diff] [blame] | 2929 | /* |
| 2930 | * Step 2 |
| 2931 | */ |
| 2932 | delta %= 1024; |
| 2933 | contrib = __accumulate_pelt_segments(periods, |
| 2934 | 1024 - sa->period_contrib, delta); |
| 2935 | } |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2936 | sa->period_contrib = delta; |
| 2937 | |
| 2938 | contrib = cap_scale(contrib, scale_freq); |
| 2939 | if (weight) { |
| 2940 | sa->load_sum += weight * contrib; |
| 2941 | if (cfs_rq) |
| 2942 | cfs_rq->runnable_load_sum += weight * contrib; |
| 2943 | } |
| 2944 | if (running) |
| 2945 | sa->util_sum += contrib * scale_cpu; |
| 2946 | |
| 2947 | return periods; |
| 2948 | } |
| 2949 | |
| 2950 | /* |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2951 | * We can represent the historical contribution to runnable average as the |
| 2952 | * coefficients of a geometric series. To do this we sub-divide our runnable |
| 2953 | * history into segments of approximately 1ms (1024us); label the segment that |
| 2954 | * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g. |
| 2955 | * |
| 2956 | * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ... |
| 2957 | * p0 p1 p2 |
| 2958 | * (now) (~1ms ago) (~2ms ago) |
| 2959 | * |
| 2960 | * Let u_i denote the fraction of p_i that the entity was runnable. |
| 2961 | * |
| 2962 | * We then designate the fractions u_i as our co-efficients, yielding the |
| 2963 | * following representation of historical load: |
| 2964 | * u_0 + u_1*y + u_2*y^2 + u_3*y^3 + ... |
| 2965 | * |
| 2966 | * We choose y based on the with of a reasonably scheduling period, fixing: |
| 2967 | * y^32 = 0.5 |
| 2968 | * |
| 2969 | * This means that the contribution to load ~32ms ago (u_32) will be weighted |
| 2970 | * approximately half as much as the contribution to load within the last ms |
| 2971 | * (u_0). |
| 2972 | * |
| 2973 | * When a period "rolls over" and we have new u_0`, multiplying the previous |
| 2974 | * sum again by y is sufficient to update: |
| 2975 | * load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... ) |
| 2976 | * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] |
| 2977 | */ |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 2978 | static __always_inline int |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 2979 | ___update_load_avg(u64 now, int cpu, struct sched_avg *sa, |
Yuyang Du | 1396223 | 2015-07-15 08:04:41 +0800 | [diff] [blame] | 2980 | unsigned long weight, int running, struct cfs_rq *cfs_rq) |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2981 | { |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 2982 | u64 delta; |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2983 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 2984 | delta = now - sa->last_update_time; |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2985 | /* |
| 2986 | * This should only happen when time goes backwards, which it |
| 2987 | * unfortunately does during sched clock init when we swap over to TSC. |
| 2988 | */ |
| 2989 | if ((s64)delta < 0) { |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 2990 | sa->last_update_time = now; |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 2991 | return 0; |
| 2992 | } |
| 2993 | |
| 2994 | /* |
| 2995 | * Use 1024ns as the unit of measurement since it's a reasonable |
| 2996 | * approximation of 1us and fast to compute. |
| 2997 | */ |
| 2998 | delta >>= 10; |
| 2999 | if (!delta) |
| 3000 | return 0; |
Peter Zijlstra | bb0bd04 | 2017-04-10 13:20:45 +0200 | [diff] [blame] | 3001 | |
| 3002 | sa->last_update_time += delta << 10; |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 3003 | |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 3004 | /* |
Vincent Guittot | f235a54 | 2017-07-01 07:06:13 +0200 | [diff] [blame] | 3005 | * running is a subset of runnable (weight) so running can't be set if |
| 3006 | * runnable is clear. But there are some corner cases where the current |
| 3007 | * se has been already dequeued but cfs_rq->curr still points to it. |
| 3008 | * This means that weight will be 0 but not running for a sched_entity |
| 3009 | * but also for a cfs_rq if the latter becomes idle. As an example, |
| 3010 | * this happens during idle_balance() which calls |
| 3011 | * update_blocked_averages() |
| 3012 | */ |
| 3013 | if (!weight) |
| 3014 | running = 0; |
| 3015 | |
| 3016 | /* |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 3017 | * Now we know we crossed measurement unit boundaries. The *_avg |
| 3018 | * accrues by two steps: |
| 3019 | * |
| 3020 | * Step 1: accumulate *_sum since last_update_time. If we haven't |
| 3021 | * crossed period boundaries, finish. |
| 3022 | */ |
| 3023 | if (!accumulate_sum(delta, cpu, sa, weight, running, cfs_rq)) |
| 3024 | return 0; |
Dietmar Eggemann | 6f2b045 | 2015-09-07 14:57:22 +0100 | [diff] [blame] | 3025 | |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 3026 | /* |
| 3027 | * Step 2: update *_avg. |
| 3028 | */ |
Vincent Guittot | 625ed2b | 2017-04-26 08:27:56 +0200 | [diff] [blame] | 3029 | sa->load_avg = div_u64(sa->load_sum, LOAD_AVG_MAX - 1024 + sa->period_contrib); |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 3030 | if (cfs_rq) { |
| 3031 | cfs_rq->runnable_load_avg = |
Vincent Guittot | 625ed2b | 2017-04-26 08:27:56 +0200 | [diff] [blame] | 3032 | div_u64(cfs_rq->runnable_load_sum, LOAD_AVG_MAX - 1024 + sa->period_contrib); |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 3033 | } |
Vincent Guittot | 625ed2b | 2017-04-26 08:27:56 +0200 | [diff] [blame] | 3034 | sa->util_avg = sa->util_sum / (LOAD_AVG_MAX - 1024 + sa->period_contrib); |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 3035 | |
Yuyang Du | a481db3 | 2017-02-13 05:44:23 +0800 | [diff] [blame] | 3036 | return 1; |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 3037 | } |
| 3038 | |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3039 | static int |
| 3040 | __update_load_avg_blocked_se(u64 now, int cpu, struct sched_entity *se) |
| 3041 | { |
| 3042 | return ___update_load_avg(now, cpu, &se->avg, 0, 0, NULL); |
| 3043 | } |
| 3044 | |
| 3045 | static int |
| 3046 | __update_load_avg_se(u64 now, int cpu, struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3047 | { |
| 3048 | return ___update_load_avg(now, cpu, &se->avg, |
| 3049 | se->on_rq * scale_load_down(se->load.weight), |
| 3050 | cfs_rq->curr == se, NULL); |
| 3051 | } |
| 3052 | |
| 3053 | static int |
| 3054 | __update_load_avg_cfs_rq(u64 now, int cpu, struct cfs_rq *cfs_rq) |
| 3055 | { |
| 3056 | return ___update_load_avg(now, cpu, &cfs_rq->avg, |
| 3057 | scale_load_down(cfs_rq->load.weight), |
| 3058 | cfs_rq->curr != NULL, cfs_rq); |
| 3059 | } |
| 3060 | |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3061 | /* |
| 3062 | * Signed add and clamp on underflow. |
| 3063 | * |
| 3064 | * Explicitly do a load-store to ensure the intermediate value never hits |
| 3065 | * memory. This allows lockless observations without ever seeing the negative |
| 3066 | * values. |
| 3067 | */ |
| 3068 | #define add_positive(_ptr, _val) do { \ |
| 3069 | typeof(_ptr) ptr = (_ptr); \ |
| 3070 | typeof(_val) val = (_val); \ |
| 3071 | typeof(*ptr) res, var = READ_ONCE(*ptr); \ |
| 3072 | \ |
| 3073 | res = var + val; \ |
| 3074 | \ |
| 3075 | if (val < 0 && res > var) \ |
| 3076 | res = 0; \ |
| 3077 | \ |
| 3078 | WRITE_ONCE(*ptr, res); \ |
| 3079 | } while (0) |
| 3080 | |
Paul Turner | c566e8e | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3081 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Peter Zijlstra | 7c3edd2 | 2016-07-13 10:56:25 +0200 | [diff] [blame] | 3082 | /** |
| 3083 | * update_tg_load_avg - update the tg's load avg |
| 3084 | * @cfs_rq: the cfs_rq whose avg changed |
| 3085 | * @force: update regardless of how small the difference |
| 3086 | * |
| 3087 | * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. |
| 3088 | * However, because tg->load_avg is a global value there are performance |
| 3089 | * considerations. |
| 3090 | * |
| 3091 | * In order to avoid having to look at the other cfs_rq's, we use a |
| 3092 | * differential update where we store the last value we propagated. This in |
| 3093 | * turn allows skipping updates if the differential is 'small'. |
| 3094 | * |
Rik van Riel | 815abf5 | 2017-06-23 12:55:30 -0400 | [diff] [blame] | 3095 | * Updating tg's load_avg is necessary before update_cfs_share(). |
Paul Turner | bb17f65 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3096 | */ |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3097 | static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) |
Paul Turner | bb17f65 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3098 | { |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3099 | long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; |
Paul Turner | bb17f65 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3100 | |
Waiman Long | aa0b7ae | 2015-12-02 13:41:50 -0500 | [diff] [blame] | 3101 | /* |
| 3102 | * No need to update load_avg for root_task_group as it is not used. |
| 3103 | */ |
| 3104 | if (cfs_rq->tg == &root_task_group) |
| 3105 | return; |
| 3106 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3107 | if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { |
| 3108 | atomic_long_add(delta, &cfs_rq->tg->load_avg); |
| 3109 | cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; |
Paul Turner | bb17f65 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3110 | } |
Paul Turner | 8165e14 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3111 | } |
Dietmar Eggemann | f5f9739 | 2014-02-26 11:19:33 +0000 | [diff] [blame] | 3112 | |
Byungchul Park | ad936d8 | 2015-10-24 01:16:19 +0900 | [diff] [blame] | 3113 | /* |
| 3114 | * Called within set_task_rq() right before setting a task's cpu. The |
| 3115 | * caller only guarantees p->pi_lock is held; no other assumptions, |
| 3116 | * including the state of rq->lock, should be made. |
| 3117 | */ |
| 3118 | void set_task_rq_fair(struct sched_entity *se, |
| 3119 | struct cfs_rq *prev, struct cfs_rq *next) |
| 3120 | { |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3121 | u64 p_last_update_time; |
| 3122 | u64 n_last_update_time; |
| 3123 | |
Byungchul Park | ad936d8 | 2015-10-24 01:16:19 +0900 | [diff] [blame] | 3124 | if (!sched_feat(ATTACH_AGE_LOAD)) |
| 3125 | return; |
| 3126 | |
| 3127 | /* |
| 3128 | * We are supposed to update the task to "current" time, then its up to |
| 3129 | * date and ready to go to new CPU/cfs_rq. But we have difficulty in |
| 3130 | * getting what current time is, so simply throw away the out-of-date |
| 3131 | * time. This will result in the wakee task is less decayed, but giving |
| 3132 | * the wakee more load sounds not bad. |
| 3133 | */ |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3134 | if (!(se->avg.last_update_time && prev)) |
| 3135 | return; |
Byungchul Park | ad936d8 | 2015-10-24 01:16:19 +0900 | [diff] [blame] | 3136 | |
| 3137 | #ifndef CONFIG_64BIT |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3138 | { |
Byungchul Park | ad936d8 | 2015-10-24 01:16:19 +0900 | [diff] [blame] | 3139 | u64 p_last_update_time_copy; |
| 3140 | u64 n_last_update_time_copy; |
| 3141 | |
| 3142 | do { |
| 3143 | p_last_update_time_copy = prev->load_last_update_time_copy; |
| 3144 | n_last_update_time_copy = next->load_last_update_time_copy; |
| 3145 | |
| 3146 | smp_rmb(); |
| 3147 | |
| 3148 | p_last_update_time = prev->avg.last_update_time; |
| 3149 | n_last_update_time = next->avg.last_update_time; |
| 3150 | |
| 3151 | } while (p_last_update_time != p_last_update_time_copy || |
| 3152 | n_last_update_time != n_last_update_time_copy); |
Byungchul Park | ad936d8 | 2015-10-24 01:16:19 +0900 | [diff] [blame] | 3153 | } |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3154 | #else |
| 3155 | p_last_update_time = prev->avg.last_update_time; |
| 3156 | n_last_update_time = next->avg.last_update_time; |
| 3157 | #endif |
| 3158 | __update_load_avg_blocked_se(p_last_update_time, cpu_of(rq_of(prev)), se); |
| 3159 | se->avg.last_update_time = n_last_update_time; |
Byungchul Park | ad936d8 | 2015-10-24 01:16:19 +0900 | [diff] [blame] | 3160 | } |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3161 | |
| 3162 | /* Take into account change of utilization of a child task group */ |
| 3163 | static inline void |
| 3164 | update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3165 | { |
| 3166 | struct cfs_rq *gcfs_rq = group_cfs_rq(se); |
| 3167 | long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; |
| 3168 | |
| 3169 | /* Nothing to update */ |
| 3170 | if (!delta) |
| 3171 | return; |
| 3172 | |
| 3173 | /* Set new sched_entity's utilization */ |
| 3174 | se->avg.util_avg = gcfs_rq->avg.util_avg; |
| 3175 | se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; |
| 3176 | |
| 3177 | /* Update parent cfs_rq utilization */ |
| 3178 | add_positive(&cfs_rq->avg.util_avg, delta); |
| 3179 | cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; |
| 3180 | } |
| 3181 | |
| 3182 | /* Take into account change of load of a child task group */ |
| 3183 | static inline void |
| 3184 | update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3185 | { |
| 3186 | struct cfs_rq *gcfs_rq = group_cfs_rq(se); |
| 3187 | long delta, load = gcfs_rq->avg.load_avg; |
| 3188 | |
| 3189 | /* |
| 3190 | * If the load of group cfs_rq is null, the load of the |
| 3191 | * sched_entity will also be null so we can skip the formula |
| 3192 | */ |
| 3193 | if (load) { |
| 3194 | long tg_load; |
| 3195 | |
| 3196 | /* Get tg's load and ensure tg_load > 0 */ |
| 3197 | tg_load = atomic_long_read(&gcfs_rq->tg->load_avg) + 1; |
| 3198 | |
| 3199 | /* Ensure tg_load >= load and updated with current load*/ |
| 3200 | tg_load -= gcfs_rq->tg_load_avg_contrib; |
| 3201 | tg_load += load; |
| 3202 | |
| 3203 | /* |
| 3204 | * We need to compute a correction term in the case that the |
| 3205 | * task group is consuming more CPU than a task of equal |
| 3206 | * weight. A task with a weight equals to tg->shares will have |
| 3207 | * a load less or equal to scale_load_down(tg->shares). |
| 3208 | * Similarly, the sched_entities that represent the task group |
| 3209 | * at parent level, can't have a load higher than |
| 3210 | * scale_load_down(tg->shares). And the Sum of sched_entities' |
| 3211 | * load must be <= scale_load_down(tg->shares). |
| 3212 | */ |
| 3213 | if (tg_load > scale_load_down(gcfs_rq->tg->shares)) { |
| 3214 | /* scale gcfs_rq's load into tg's shares*/ |
| 3215 | load *= scale_load_down(gcfs_rq->tg->shares); |
| 3216 | load /= tg_load; |
| 3217 | } |
| 3218 | } |
| 3219 | |
| 3220 | delta = load - se->avg.load_avg; |
| 3221 | |
| 3222 | /* Nothing to update */ |
| 3223 | if (!delta) |
| 3224 | return; |
| 3225 | |
| 3226 | /* Set new sched_entity's load */ |
| 3227 | se->avg.load_avg = load; |
| 3228 | se->avg.load_sum = se->avg.load_avg * LOAD_AVG_MAX; |
| 3229 | |
| 3230 | /* Update parent cfs_rq load */ |
| 3231 | add_positive(&cfs_rq->avg.load_avg, delta); |
| 3232 | cfs_rq->avg.load_sum = cfs_rq->avg.load_avg * LOAD_AVG_MAX; |
| 3233 | |
| 3234 | /* |
| 3235 | * If the sched_entity is already enqueued, we also have to update the |
| 3236 | * runnable load avg. |
| 3237 | */ |
| 3238 | if (se->on_rq) { |
| 3239 | /* Update parent cfs_rq runnable_load_avg */ |
| 3240 | add_positive(&cfs_rq->runnable_load_avg, delta); |
| 3241 | cfs_rq->runnable_load_sum = cfs_rq->runnable_load_avg * LOAD_AVG_MAX; |
| 3242 | } |
| 3243 | } |
| 3244 | |
| 3245 | static inline void set_tg_cfs_propagate(struct cfs_rq *cfs_rq) |
| 3246 | { |
| 3247 | cfs_rq->propagate_avg = 1; |
| 3248 | } |
| 3249 | |
| 3250 | static inline int test_and_clear_tg_cfs_propagate(struct sched_entity *se) |
| 3251 | { |
| 3252 | struct cfs_rq *cfs_rq = group_cfs_rq(se); |
| 3253 | |
| 3254 | if (!cfs_rq->propagate_avg) |
| 3255 | return 0; |
| 3256 | |
| 3257 | cfs_rq->propagate_avg = 0; |
| 3258 | return 1; |
| 3259 | } |
| 3260 | |
| 3261 | /* Update task and its cfs_rq load average */ |
| 3262 | static inline int propagate_entity_load_avg(struct sched_entity *se) |
| 3263 | { |
| 3264 | struct cfs_rq *cfs_rq; |
| 3265 | |
| 3266 | if (entity_is_task(se)) |
| 3267 | return 0; |
| 3268 | |
| 3269 | if (!test_and_clear_tg_cfs_propagate(se)) |
| 3270 | return 0; |
| 3271 | |
| 3272 | cfs_rq = cfs_rq_of(se); |
| 3273 | |
| 3274 | set_tg_cfs_propagate(cfs_rq); |
| 3275 | |
| 3276 | update_tg_cfs_util(cfs_rq, se); |
| 3277 | update_tg_cfs_load(cfs_rq, se); |
| 3278 | |
| 3279 | return 1; |
| 3280 | } |
| 3281 | |
Vincent Guittot | bc42789 | 2017-03-17 14:47:22 +0100 | [diff] [blame] | 3282 | /* |
| 3283 | * Check if we need to update the load and the utilization of a blocked |
| 3284 | * group_entity: |
| 3285 | */ |
| 3286 | static inline bool skip_blocked_update(struct sched_entity *se) |
| 3287 | { |
| 3288 | struct cfs_rq *gcfs_rq = group_cfs_rq(se); |
| 3289 | |
| 3290 | /* |
| 3291 | * If sched_entity still have not zero load or utilization, we have to |
| 3292 | * decay it: |
| 3293 | */ |
| 3294 | if (se->avg.load_avg || se->avg.util_avg) |
| 3295 | return false; |
| 3296 | |
| 3297 | /* |
| 3298 | * If there is a pending propagation, we have to update the load and |
| 3299 | * the utilization of the sched_entity: |
| 3300 | */ |
| 3301 | if (gcfs_rq->propagate_avg) |
| 3302 | return false; |
| 3303 | |
| 3304 | /* |
| 3305 | * Otherwise, the load and the utilization of the sched_entity is |
| 3306 | * already zero and there is no pending propagation, so it will be a |
| 3307 | * waste of time to try to decay it: |
| 3308 | */ |
| 3309 | return true; |
| 3310 | } |
| 3311 | |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 3312 | #else /* CONFIG_FAIR_GROUP_SCHED */ |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3313 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3314 | static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {} |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3315 | |
| 3316 | static inline int propagate_entity_load_avg(struct sched_entity *se) |
| 3317 | { |
| 3318 | return 0; |
| 3319 | } |
| 3320 | |
| 3321 | static inline void set_tg_cfs_propagate(struct cfs_rq *cfs_rq) {} |
| 3322 | |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 3323 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
Paul Turner | c566e8e | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3324 | |
Peter Zijlstra | 8974189 | 2016-06-16 10:50:40 +0200 | [diff] [blame] | 3325 | /* |
| 3326 | * Unsigned subtract and clamp on underflow. |
| 3327 | * |
| 3328 | * Explicitly do a load-store to ensure the intermediate value never hits |
| 3329 | * memory. This allows lockless observations without ever seeing the negative |
| 3330 | * values. |
| 3331 | */ |
| 3332 | #define sub_positive(_ptr, _val) do { \ |
| 3333 | typeof(_ptr) ptr = (_ptr); \ |
| 3334 | typeof(*ptr) val = (_val); \ |
| 3335 | typeof(*ptr) res, var = READ_ONCE(*ptr); \ |
| 3336 | res = var - val; \ |
| 3337 | if (res > var) \ |
| 3338 | res = 0; \ |
| 3339 | WRITE_ONCE(*ptr, res); \ |
| 3340 | } while (0) |
| 3341 | |
Peter Zijlstra | 3d30544f | 2016-06-21 14:27:50 +0200 | [diff] [blame] | 3342 | /** |
| 3343 | * update_cfs_rq_load_avg - update the cfs_rq's load/util averages |
| 3344 | * @now: current time, as per cfs_rq_clock_task() |
| 3345 | * @cfs_rq: cfs_rq to update |
Peter Zijlstra | 3d30544f | 2016-06-21 14:27:50 +0200 | [diff] [blame] | 3346 | * |
| 3347 | * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) |
| 3348 | * avg. The immediate corollary is that all (fair) tasks must be attached, see |
| 3349 | * post_init_entity_util_avg(). |
| 3350 | * |
| 3351 | * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. |
| 3352 | * |
Peter Zijlstra | 7c3edd2 | 2016-07-13 10:56:25 +0200 | [diff] [blame] | 3353 | * Returns true if the load decayed or we removed load. |
| 3354 | * |
| 3355 | * Since both these conditions indicate a changed cfs_rq->avg.load we should |
| 3356 | * call update_tg_load_avg() when this function returns true. |
Peter Zijlstra | 3d30544f | 2016-06-21 14:27:50 +0200 | [diff] [blame] | 3357 | */ |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3358 | static inline int |
Viresh Kumar | 3a123bb | 2017-05-24 10:59:56 +0530 | [diff] [blame] | 3359 | update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3360 | { |
| 3361 | struct sched_avg *sa = &cfs_rq->avg; |
| 3362 | int decayed, removed_load = 0, removed_util = 0; |
| 3363 | |
| 3364 | if (atomic_long_read(&cfs_rq->removed_load_avg)) { |
| 3365 | s64 r = atomic_long_xchg(&cfs_rq->removed_load_avg, 0); |
Peter Zijlstra | 8974189 | 2016-06-16 10:50:40 +0200 | [diff] [blame] | 3366 | sub_positive(&sa->load_avg, r); |
| 3367 | sub_positive(&sa->load_sum, r * LOAD_AVG_MAX); |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3368 | removed_load = 1; |
Vincent Guittot | 4e51607 | 2016-11-08 10:53:46 +0100 | [diff] [blame] | 3369 | set_tg_cfs_propagate(cfs_rq); |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3370 | } |
| 3371 | |
| 3372 | if (atomic_long_read(&cfs_rq->removed_util_avg)) { |
| 3373 | long r = atomic_long_xchg(&cfs_rq->removed_util_avg, 0); |
Peter Zijlstra | 8974189 | 2016-06-16 10:50:40 +0200 | [diff] [blame] | 3374 | sub_positive(&sa->util_avg, r); |
| 3375 | sub_positive(&sa->util_sum, r * LOAD_AVG_MAX); |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3376 | removed_util = 1; |
Vincent Guittot | 4e51607 | 2016-11-08 10:53:46 +0100 | [diff] [blame] | 3377 | set_tg_cfs_propagate(cfs_rq); |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3378 | } |
| 3379 | |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3380 | decayed = __update_load_avg_cfs_rq(now, cpu_of(rq_of(cfs_rq)), cfs_rq); |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3381 | |
| 3382 | #ifndef CONFIG_64BIT |
| 3383 | smp_wmb(); |
| 3384 | cfs_rq->load_last_update_time_copy = sa->last_update_time; |
| 3385 | #endif |
| 3386 | |
Viresh Kumar | 3a123bb | 2017-05-24 10:59:56 +0530 | [diff] [blame] | 3387 | if (decayed || removed_util) |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3388 | cfs_rq_util_change(cfs_rq); |
Steve Muckle | 21e96f8 | 2016-03-21 17:21:07 -0700 | [diff] [blame] | 3389 | |
Steve Muckle | 41e0d37 | 2016-03-21 17:21:08 -0700 | [diff] [blame] | 3390 | return decayed || removed_load; |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3391 | } |
| 3392 | |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3393 | /* |
| 3394 | * Optional action to be done while updating the load average |
| 3395 | */ |
| 3396 | #define UPDATE_TG 0x1 |
| 3397 | #define SKIP_AGE_LOAD 0x2 |
| 3398 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3399 | /* Update task and its cfs_rq load average */ |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3400 | static inline void update_load_avg(struct sched_entity *se, int flags) |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 3401 | { |
Paul Turner | 2dac754 | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3402 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3403 | u64 now = cfs_rq_clock_task(cfs_rq); |
Rafael J. Wysocki | 34e2c55 | 2016-02-15 20:20:42 +0100 | [diff] [blame] | 3404 | struct rq *rq = rq_of(cfs_rq); |
| 3405 | int cpu = cpu_of(rq); |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3406 | int decayed; |
Paul Turner | 2dac754 | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3407 | |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3408 | /* |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3409 | * Track task load average for carrying it to new CPU after migrated, and |
| 3410 | * track group sched_entity load average for task_h_load calc in migration |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3411 | */ |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3412 | if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) |
| 3413 | __update_load_avg_se(now, cpu, cfs_rq, se); |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 3414 | |
Viresh Kumar | 3a123bb | 2017-05-24 10:59:56 +0530 | [diff] [blame] | 3415 | decayed = update_cfs_rq_load_avg(now, cfs_rq); |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3416 | decayed |= propagate_entity_load_avg(se); |
| 3417 | |
| 3418 | if (decayed && (flags & UPDATE_TG)) |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3419 | update_tg_load_avg(cfs_rq, 0); |
| 3420 | } |
Paul Turner | 2dac754 | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3421 | |
Peter Zijlstra | 3d30544f | 2016-06-21 14:27:50 +0200 | [diff] [blame] | 3422 | /** |
| 3423 | * attach_entity_load_avg - attach this entity to its cfs_rq load avg |
| 3424 | * @cfs_rq: cfs_rq to attach to |
| 3425 | * @se: sched_entity to attach |
| 3426 | * |
| 3427 | * Must call update_cfs_rq_load_avg() before this, since we rely on |
| 3428 | * cfs_rq->avg.last_update_time being current. |
| 3429 | */ |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3430 | static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3431 | { |
| 3432 | se->avg.last_update_time = cfs_rq->avg.last_update_time; |
| 3433 | cfs_rq->avg.load_avg += se->avg.load_avg; |
| 3434 | cfs_rq->avg.load_sum += se->avg.load_sum; |
| 3435 | cfs_rq->avg.util_avg += se->avg.util_avg; |
| 3436 | cfs_rq->avg.util_sum += se->avg.util_sum; |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3437 | set_tg_cfs_propagate(cfs_rq); |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3438 | |
| 3439 | cfs_rq_util_change(cfs_rq); |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3440 | } |
| 3441 | |
Peter Zijlstra | 3d30544f | 2016-06-21 14:27:50 +0200 | [diff] [blame] | 3442 | /** |
| 3443 | * detach_entity_load_avg - detach this entity from its cfs_rq load avg |
| 3444 | * @cfs_rq: cfs_rq to detach from |
| 3445 | * @se: sched_entity to detach |
| 3446 | * |
| 3447 | * Must call update_cfs_rq_load_avg() before this, since we rely on |
| 3448 | * cfs_rq->avg.last_update_time being current. |
| 3449 | */ |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3450 | static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3451 | { |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3452 | |
Peter Zijlstra | 8974189 | 2016-06-16 10:50:40 +0200 | [diff] [blame] | 3453 | sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); |
| 3454 | sub_positive(&cfs_rq->avg.load_sum, se->avg.load_sum); |
| 3455 | sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); |
| 3456 | sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 3457 | set_tg_cfs_propagate(cfs_rq); |
Steve Muckle | a2c6c91 | 2016-03-24 15:26:07 -0700 | [diff] [blame] | 3458 | |
| 3459 | cfs_rq_util_change(cfs_rq); |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3460 | } |
| 3461 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3462 | /* Add the load generated by se into cfs_rq's load average */ |
| 3463 | static inline void |
| 3464 | enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3465 | { |
| 3466 | struct sched_avg *sa = &se->avg; |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3467 | |
Yuyang Du | 1396223 | 2015-07-15 08:04:41 +0800 | [diff] [blame] | 3468 | cfs_rq->runnable_load_avg += sa->load_avg; |
| 3469 | cfs_rq->runnable_load_sum += sa->load_sum; |
| 3470 | |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3471 | if (!sa->last_update_time) { |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3472 | attach_entity_load_avg(cfs_rq, se); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3473 | update_tg_load_avg(cfs_rq, 0); |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3474 | } |
Paul Turner | 9ee474f | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3475 | } |
| 3476 | |
Yuyang Du | 1396223 | 2015-07-15 08:04:41 +0800 | [diff] [blame] | 3477 | /* Remove the runnable load generated by se from cfs_rq's runnable load average */ |
| 3478 | static inline void |
| 3479 | dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3480 | { |
Yuyang Du | 1396223 | 2015-07-15 08:04:41 +0800 | [diff] [blame] | 3481 | cfs_rq->runnable_load_avg = |
| 3482 | max_t(long, cfs_rq->runnable_load_avg - se->avg.load_avg, 0); |
| 3483 | cfs_rq->runnable_load_sum = |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3484 | max_t(s64, cfs_rq->runnable_load_sum - se->avg.load_sum, 0); |
Yuyang Du | 1396223 | 2015-07-15 08:04:41 +0800 | [diff] [blame] | 3485 | } |
| 3486 | |
Yuyang Du | 0905f04 | 2015-12-17 07:34:27 +0800 | [diff] [blame] | 3487 | #ifndef CONFIG_64BIT |
| 3488 | static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) |
| 3489 | { |
| 3490 | u64 last_update_time_copy; |
| 3491 | u64 last_update_time; |
| 3492 | |
| 3493 | do { |
| 3494 | last_update_time_copy = cfs_rq->load_last_update_time_copy; |
| 3495 | smp_rmb(); |
| 3496 | last_update_time = cfs_rq->avg.last_update_time; |
| 3497 | } while (last_update_time != last_update_time_copy); |
| 3498 | |
| 3499 | return last_update_time; |
| 3500 | } |
| 3501 | #else |
| 3502 | static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) |
| 3503 | { |
| 3504 | return cfs_rq->avg.last_update_time; |
| 3505 | } |
| 3506 | #endif |
| 3507 | |
Paul Turner | 9ee474f | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3508 | /* |
Morten Rasmussen | 104cb16 | 2016-10-14 14:41:07 +0100 | [diff] [blame] | 3509 | * Synchronize entity load avg of dequeued entity without locking |
| 3510 | * the previous rq. |
| 3511 | */ |
| 3512 | void sync_entity_load_avg(struct sched_entity *se) |
| 3513 | { |
| 3514 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 3515 | u64 last_update_time; |
| 3516 | |
| 3517 | last_update_time = cfs_rq_last_update_time(cfs_rq); |
Peter Zijlstra | 0ccb977 | 2017-03-28 11:08:20 +0200 | [diff] [blame] | 3518 | __update_load_avg_blocked_se(last_update_time, cpu_of(rq_of(cfs_rq)), se); |
Morten Rasmussen | 104cb16 | 2016-10-14 14:41:07 +0100 | [diff] [blame] | 3519 | } |
| 3520 | |
| 3521 | /* |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3522 | * Task first catches up with cfs_rq, and then subtract |
| 3523 | * itself from the cfs_rq (task must be off the queue now). |
Paul Turner | 9ee474f | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3524 | */ |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3525 | void remove_entity_load_avg(struct sched_entity *se) |
Paul Turner | 9ee474f | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3526 | { |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3527 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
Paul Turner | 9ee474f | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3528 | |
Yuyang Du | 0905f04 | 2015-12-17 07:34:27 +0800 | [diff] [blame] | 3529 | /* |
Peter Zijlstra | 7dc603c | 2016-06-16 13:29:28 +0200 | [diff] [blame] | 3530 | * tasks cannot exit without having gone through wake_up_new_task() -> |
| 3531 | * post_init_entity_util_avg() which will have added things to the |
| 3532 | * cfs_rq, so we can remove unconditionally. |
| 3533 | * |
| 3534 | * Similarly for groups, they will have passed through |
| 3535 | * post_init_entity_util_avg() before unregister_sched_fair_group() |
| 3536 | * calls this. |
Yuyang Du | 0905f04 | 2015-12-17 07:34:27 +0800 | [diff] [blame] | 3537 | */ |
Paul Turner | 9ee474f | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3538 | |
Morten Rasmussen | 104cb16 | 2016-10-14 14:41:07 +0100 | [diff] [blame] | 3539 | sync_entity_load_avg(se); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3540 | atomic_long_add(se->avg.load_avg, &cfs_rq->removed_load_avg); |
| 3541 | atomic_long_add(se->avg.util_avg, &cfs_rq->removed_util_avg); |
Paul Turner | 2dac754 | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 3542 | } |
Vincent Guittot | 642dbc3 | 2013-04-18 18:34:26 +0200 | [diff] [blame] | 3543 | |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 3544 | static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq) |
| 3545 | { |
| 3546 | return cfs_rq->runnable_load_avg; |
| 3547 | } |
| 3548 | |
| 3549 | static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) |
| 3550 | { |
| 3551 | return cfs_rq->avg.load_avg; |
| 3552 | } |
| 3553 | |
Matt Fleming | 46f69fa | 2016-09-21 14:38:12 +0100 | [diff] [blame] | 3554 | static int idle_balance(struct rq *this_rq, struct rq_flags *rf); |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 3555 | |
Peter Zijlstra | 38033c3 | 2014-01-23 20:32:21 +0100 | [diff] [blame] | 3556 | #else /* CONFIG_SMP */ |
| 3557 | |
Peter Zijlstra | 0101147 | 2016-06-17 11:20:46 +0200 | [diff] [blame] | 3558 | static inline int |
Viresh Kumar | 3a123bb | 2017-05-24 10:59:56 +0530 | [diff] [blame] | 3559 | update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) |
Peter Zijlstra | 0101147 | 2016-06-17 11:20:46 +0200 | [diff] [blame] | 3560 | { |
| 3561 | return 0; |
| 3562 | } |
| 3563 | |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3564 | #define UPDATE_TG 0x0 |
| 3565 | #define SKIP_AGE_LOAD 0x0 |
| 3566 | |
| 3567 | static inline void update_load_avg(struct sched_entity *se, int not_used1) |
Rafael J. Wysocki | 536bd00 | 2016-05-06 14:58:43 +0200 | [diff] [blame] | 3568 | { |
Viresh Kumar | a030d73 | 2017-05-24 10:59:52 +0530 | [diff] [blame] | 3569 | cfs_rq_util_change(cfs_rq_of(se)); |
Rafael J. Wysocki | 536bd00 | 2016-05-06 14:58:43 +0200 | [diff] [blame] | 3570 | } |
| 3571 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3572 | static inline void |
| 3573 | enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} |
Yuyang Du | 1396223 | 2015-07-15 08:04:41 +0800 | [diff] [blame] | 3574 | static inline void |
| 3575 | dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3576 | static inline void remove_entity_load_avg(struct sched_entity *se) {} |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 3577 | |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 3578 | static inline void |
| 3579 | attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} |
| 3580 | static inline void |
| 3581 | detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} |
| 3582 | |
Matt Fleming | 46f69fa | 2016-09-21 14:38:12 +0100 | [diff] [blame] | 3583 | static inline int idle_balance(struct rq *rq, struct rq_flags *rf) |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 3584 | { |
| 3585 | return 0; |
| 3586 | } |
| 3587 | |
Peter Zijlstra | 38033c3 | 2014-01-23 20:32:21 +0100 | [diff] [blame] | 3588 | #endif /* CONFIG_SMP */ |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 3589 | |
Peter Zijlstra | ddc9729 | 2007-10-15 17:00:10 +0200 | [diff] [blame] | 3590 | static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3591 | { |
| 3592 | #ifdef CONFIG_SCHED_DEBUG |
| 3593 | s64 d = se->vruntime - cfs_rq->min_vruntime; |
| 3594 | |
| 3595 | if (d < 0) |
| 3596 | d = -d; |
| 3597 | |
| 3598 | if (d > 3*sysctl_sched_latency) |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 3599 | schedstat_inc(cfs_rq->nr_spread_over); |
Peter Zijlstra | ddc9729 | 2007-10-15 17:00:10 +0200 | [diff] [blame] | 3600 | #endif |
| 3601 | } |
| 3602 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3603 | static void |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3604 | place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) |
| 3605 | { |
Peter Zijlstra | 1af5f73 | 2008-10-24 11:06:13 +0200 | [diff] [blame] | 3606 | u64 vruntime = cfs_rq->min_vruntime; |
Peter Zijlstra | 94dfb5e | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3607 | |
Peter Zijlstra | 2cb8600 | 2007-11-09 22:39:37 +0100 | [diff] [blame] | 3608 | /* |
| 3609 | * The 'current' period is already promised to the current tasks, |
| 3610 | * however the extra weight of the new task will slow them down a |
| 3611 | * little, place the new task so that it fits in the slot that |
| 3612 | * stays open at the end. |
| 3613 | */ |
Peter Zijlstra | 94dfb5e | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3614 | if (initial && sched_feat(START_DEBIT)) |
Peter Zijlstra | f9c0b09 | 2008-10-17 19:27:04 +0200 | [diff] [blame] | 3615 | vruntime += sched_vslice(cfs_rq, se); |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3616 | |
Mike Galbraith | a2e7a7e | 2009-09-18 09:19:25 +0200 | [diff] [blame] | 3617 | /* sleeps up to a single latency don't count. */ |
Mike Galbraith | 5ca9880 | 2010-03-11 17:17:17 +0100 | [diff] [blame] | 3618 | if (!initial) { |
Mike Galbraith | a2e7a7e | 2009-09-18 09:19:25 +0200 | [diff] [blame] | 3619 | unsigned long thresh = sysctl_sched_latency; |
Peter Zijlstra | a7be37a | 2008-06-27 13:41:11 +0200 | [diff] [blame] | 3620 | |
Mike Galbraith | a2e7a7e | 2009-09-18 09:19:25 +0200 | [diff] [blame] | 3621 | /* |
Mike Galbraith | a2e7a7e | 2009-09-18 09:19:25 +0200 | [diff] [blame] | 3622 | * Halve their sleep time's effect, to allow |
| 3623 | * for a gentler effect of sleepers: |
| 3624 | */ |
| 3625 | if (sched_feat(GENTLE_FAIR_SLEEPERS)) |
| 3626 | thresh >>= 1; |
Ingo Molnar | 51e0304 | 2009-09-16 08:54:45 +0200 | [diff] [blame] | 3627 | |
Mike Galbraith | a2e7a7e | 2009-09-18 09:19:25 +0200 | [diff] [blame] | 3628 | vruntime -= thresh; |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3629 | } |
| 3630 | |
Mike Galbraith | b5d9d73 | 2009-09-08 11:12:28 +0200 | [diff] [blame] | 3631 | /* ensure we never gain time by being placed backwards. */ |
Viresh Kumar | 16c8f1c | 2012-11-08 13:33:46 +0530 | [diff] [blame] | 3632 | se->vruntime = max_vruntime(se->vruntime, vruntime); |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3633 | } |
| 3634 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 3635 | static void check_enqueue_throttle(struct cfs_rq *cfs_rq); |
| 3636 | |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 3637 | static inline void check_schedstat_required(void) |
| 3638 | { |
| 3639 | #ifdef CONFIG_SCHEDSTATS |
| 3640 | if (schedstat_enabled()) |
| 3641 | return; |
| 3642 | |
| 3643 | /* Force schedstat enabled if a dependent tracepoint is active */ |
| 3644 | if (trace_sched_stat_wait_enabled() || |
| 3645 | trace_sched_stat_sleep_enabled() || |
| 3646 | trace_sched_stat_iowait_enabled() || |
| 3647 | trace_sched_stat_blocked_enabled() || |
| 3648 | trace_sched_stat_runtime_enabled()) { |
Josh Poimboeuf | eda8dca | 2016-06-13 02:32:09 -0500 | [diff] [blame] | 3649 | printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, " |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 3650 | "stat_blocked and stat_runtime require the " |
Marcin Nowakowski | f67abed | 2017-06-09 10:00:29 +0200 | [diff] [blame] | 3651 | "kernel parameter schedstats=enable or " |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 3652 | "kernel.sched_schedstats=1\n"); |
| 3653 | } |
| 3654 | #endif |
| 3655 | } |
| 3656 | |
Peter Zijlstra | b5179ac | 2016-05-11 16:10:34 +0200 | [diff] [blame] | 3657 | |
| 3658 | /* |
| 3659 | * MIGRATION |
| 3660 | * |
| 3661 | * dequeue |
| 3662 | * update_curr() |
| 3663 | * update_min_vruntime() |
| 3664 | * vruntime -= min_vruntime |
| 3665 | * |
| 3666 | * enqueue |
| 3667 | * update_curr() |
| 3668 | * update_min_vruntime() |
| 3669 | * vruntime += min_vruntime |
| 3670 | * |
| 3671 | * this way the vruntime transition between RQs is done when both |
| 3672 | * min_vruntime are up-to-date. |
| 3673 | * |
| 3674 | * WAKEUP (remote) |
| 3675 | * |
Peter Zijlstra | 59efa0b | 2016-05-10 18:24:37 +0200 | [diff] [blame] | 3676 | * ->migrate_task_rq_fair() (p->state == TASK_WAKING) |
Peter Zijlstra | b5179ac | 2016-05-11 16:10:34 +0200 | [diff] [blame] | 3677 | * vruntime -= min_vruntime |
| 3678 | * |
| 3679 | * enqueue |
| 3680 | * update_curr() |
| 3681 | * update_min_vruntime() |
| 3682 | * vruntime += min_vruntime |
| 3683 | * |
| 3684 | * this way we don't have the most up-to-date min_vruntime on the originating |
| 3685 | * CPU and an up-to-date min_vruntime on the destination CPU. |
| 3686 | */ |
| 3687 | |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3688 | static void |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 3689 | enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3690 | { |
Peter Zijlstra | 2f95035 | 2016-05-11 19:27:56 +0200 | [diff] [blame] | 3691 | bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); |
| 3692 | bool curr = cfs_rq->curr == se; |
Peter Zijlstra | 3a47d51 | 2016-03-09 13:04:03 +0100 | [diff] [blame] | 3693 | |
Ingo Molnar | 53d3bc7 | 2016-05-11 08:25:53 +0200 | [diff] [blame] | 3694 | /* |
Peter Zijlstra | 2f95035 | 2016-05-11 19:27:56 +0200 | [diff] [blame] | 3695 | * If we're the current task, we must renormalise before calling |
| 3696 | * update_curr(). |
Ingo Molnar | 53d3bc7 | 2016-05-11 08:25:53 +0200 | [diff] [blame] | 3697 | */ |
Peter Zijlstra | 2f95035 | 2016-05-11 19:27:56 +0200 | [diff] [blame] | 3698 | if (renorm && curr) |
| 3699 | se->vruntime += cfs_rq->min_vruntime; |
| 3700 | |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 3701 | update_curr(cfs_rq); |
Peter Zijlstra | 2f95035 | 2016-05-11 19:27:56 +0200 | [diff] [blame] | 3702 | |
| 3703 | /* |
| 3704 | * Otherwise, renormalise after, such that we're placed at the current |
| 3705 | * moment in time, instead of some random moment in the past. Being |
| 3706 | * placed in the past could significantly boost this task to the |
| 3707 | * fairness detriment of existing tasks. |
| 3708 | */ |
| 3709 | if (renorm && !curr) |
| 3710 | se->vruntime += cfs_rq->min_vruntime; |
| 3711 | |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 3712 | /* |
| 3713 | * When enqueuing a sched_entity, we must: |
| 3714 | * - Update loads to have both entity and cfs_rq synced with now. |
| 3715 | * - Add its load to cfs_rq->runnable_avg |
| 3716 | * - For group_entity, update its weight to reflect the new share of |
| 3717 | * its group cfs_rq |
| 3718 | * - Add its new weight to cfs_rq->load.weight |
| 3719 | */ |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3720 | update_load_avg(se, UPDATE_TG); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3721 | enqueue_entity_load_avg(cfs_rq, se); |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 3722 | update_cfs_shares(se); |
Linus Torvalds | 17bc14b | 2012-12-14 07:20:43 -0800 | [diff] [blame] | 3723 | account_entity_enqueue(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3724 | |
Josh Poimboeuf | 1a3d027 | 2016-06-17 12:43:23 -0500 | [diff] [blame] | 3725 | if (flags & ENQUEUE_WAKEUP) |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3726 | place_entity(cfs_rq, se, 0); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3727 | |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 3728 | check_schedstat_required(); |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3729 | update_stats_enqueue(cfs_rq, se, flags); |
| 3730 | check_spread(cfs_rq, se); |
Peter Zijlstra | 2f95035 | 2016-05-11 19:27:56 +0200 | [diff] [blame] | 3731 | if (!curr) |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 3732 | __enqueue_entity(cfs_rq, se); |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 3733 | se->on_rq = 1; |
Peter Zijlstra | 3d4b47b | 2010-11-15 15:47:01 -0800 | [diff] [blame] | 3734 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 3735 | if (cfs_rq->nr_running == 1) { |
Peter Zijlstra | 3d4b47b | 2010-11-15 15:47:01 -0800 | [diff] [blame] | 3736 | list_add_leaf_cfs_rq(cfs_rq); |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 3737 | check_enqueue_throttle(cfs_rq); |
| 3738 | } |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3739 | } |
| 3740 | |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3741 | static void __clear_buddies_last(struct sched_entity *se) |
Peter Zijlstra | 2002c69 | 2008-11-11 11:52:33 +0100 | [diff] [blame] | 3742 | { |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3743 | for_each_sched_entity(se) { |
| 3744 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
Peter Zijlstra | f104479 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3745 | if (cfs_rq->last != se) |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3746 | break; |
Peter Zijlstra | f104479 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3747 | |
| 3748 | cfs_rq->last = NULL; |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3749 | } |
| 3750 | } |
Peter Zijlstra | 2002c69 | 2008-11-11 11:52:33 +0100 | [diff] [blame] | 3751 | |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3752 | static void __clear_buddies_next(struct sched_entity *se) |
| 3753 | { |
| 3754 | for_each_sched_entity(se) { |
| 3755 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
Peter Zijlstra | f104479 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3756 | if (cfs_rq->next != se) |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3757 | break; |
Peter Zijlstra | f104479 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3758 | |
| 3759 | cfs_rq->next = NULL; |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3760 | } |
Peter Zijlstra | 2002c69 | 2008-11-11 11:52:33 +0100 | [diff] [blame] | 3761 | } |
| 3762 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3763 | static void __clear_buddies_skip(struct sched_entity *se) |
| 3764 | { |
| 3765 | for_each_sched_entity(se) { |
| 3766 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
Peter Zijlstra | f104479 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3767 | if (cfs_rq->skip != se) |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3768 | break; |
Peter Zijlstra | f104479 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3769 | |
| 3770 | cfs_rq->skip = NULL; |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3771 | } |
| 3772 | } |
| 3773 | |
Peter Zijlstra | a571bbe | 2009-01-28 14:51:40 +0100 | [diff] [blame] | 3774 | static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 3775 | { |
Rik van Riel | 2c13c919 | 2011-02-01 09:48:37 -0500 | [diff] [blame] | 3776 | if (cfs_rq->last == se) |
| 3777 | __clear_buddies_last(se); |
| 3778 | |
| 3779 | if (cfs_rq->next == se) |
| 3780 | __clear_buddies_next(se); |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3781 | |
| 3782 | if (cfs_rq->skip == se) |
| 3783 | __clear_buddies_skip(se); |
Peter Zijlstra | a571bbe | 2009-01-28 14:51:40 +0100 | [diff] [blame] | 3784 | } |
| 3785 | |
Peter Zijlstra | 6c16a6d | 2012-03-21 13:07:16 -0700 | [diff] [blame] | 3786 | static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 3787 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3788 | static void |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 3789 | dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3790 | { |
Dmitry Adamushko | a2a2d68 | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 3791 | /* |
| 3792 | * Update run-time statistics of the 'current'. |
| 3793 | */ |
| 3794 | update_curr(cfs_rq); |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 3795 | |
| 3796 | /* |
| 3797 | * When dequeuing a sched_entity, we must: |
| 3798 | * - Update loads to have both entity and cfs_rq synced with now. |
| 3799 | * - Substract its load from the cfs_rq->runnable_avg. |
| 3800 | * - Substract its previous weight from cfs_rq->load.weight. |
| 3801 | * - For group entity, update its weight to reflect the new share |
| 3802 | * of its group cfs_rq. |
| 3803 | */ |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3804 | update_load_avg(se, UPDATE_TG); |
Yuyang Du | 1396223 | 2015-07-15 08:04:41 +0800 | [diff] [blame] | 3805 | dequeue_entity_load_avg(cfs_rq, se); |
Dmitry Adamushko | a2a2d68 | 2007-10-15 17:00:13 +0200 | [diff] [blame] | 3806 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3807 | update_stats_dequeue(cfs_rq, se, flags); |
Peter Zijlstra | 67e9fb2 | 2007-10-15 17:00:10 +0200 | [diff] [blame] | 3808 | |
Peter Zijlstra | 2002c69 | 2008-11-11 11:52:33 +0100 | [diff] [blame] | 3809 | clear_buddies(cfs_rq, se); |
Peter Zijlstra | 4793241 | 2008-11-04 21:25:09 +0100 | [diff] [blame] | 3810 | |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 3811 | if (se != cfs_rq->curr) |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 3812 | __dequeue_entity(cfs_rq, se); |
Linus Torvalds | 17bc14b | 2012-12-14 07:20:43 -0800 | [diff] [blame] | 3813 | se->on_rq = 0; |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 3814 | account_entity_dequeue(cfs_rq, se); |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 3815 | |
| 3816 | /* |
Peter Zijlstra | b60205c | 2016-09-20 21:58:12 +0200 | [diff] [blame] | 3817 | * Normalize after update_curr(); which will also have moved |
| 3818 | * min_vruntime if @se is the one holding it back. But before doing |
| 3819 | * update_min_vruntime() again, which will discount @se's position and |
| 3820 | * can move min_vruntime forward still more. |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 3821 | */ |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 3822 | if (!(flags & DEQUEUE_SLEEP)) |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 3823 | se->vruntime -= cfs_rq->min_vruntime; |
Peter Zijlstra | 1e87623 | 2011-05-17 16:21:10 -0700 | [diff] [blame] | 3824 | |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 3825 | /* return excess runtime on last dequeue */ |
| 3826 | return_cfs_rq_runtime(cfs_rq); |
| 3827 | |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 3828 | update_cfs_shares(se); |
Peter Zijlstra | b60205c | 2016-09-20 21:58:12 +0200 | [diff] [blame] | 3829 | |
| 3830 | /* |
| 3831 | * Now advance min_vruntime if @se was the entity holding it back, |
| 3832 | * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be |
| 3833 | * put back on, and if we advance min_vruntime, we'll be placed back |
| 3834 | * further than we started -- ie. we'll be penalized. |
| 3835 | */ |
| 3836 | if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) |
| 3837 | update_min_vruntime(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3838 | } |
| 3839 | |
| 3840 | /* |
| 3841 | * Preempt the current task with a newly woken task if needed: |
| 3842 | */ |
Peter Zijlstra | 7c92e54 | 2007-09-05 14:32:49 +0200 | [diff] [blame] | 3843 | static void |
Ingo Molnar | 2e09bf5 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3844 | check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3845 | { |
Peter Zijlstra | 1169783 | 2007-09-05 14:32:49 +0200 | [diff] [blame] | 3846 | unsigned long ideal_runtime, delta_exec; |
Wang Xingchao | f4cfb33 | 2011-09-16 13:35:52 -0400 | [diff] [blame] | 3847 | struct sched_entity *se; |
| 3848 | s64 delta; |
Peter Zijlstra | 1169783 | 2007-09-05 14:32:49 +0200 | [diff] [blame] | 3849 | |
Peter Zijlstra | 6d0f0eb | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 3850 | ideal_runtime = sched_slice(cfs_rq, curr); |
Peter Zijlstra | 1169783 | 2007-09-05 14:32:49 +0200 | [diff] [blame] | 3851 | delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; |
Mike Galbraith | a9f3e2b | 2009-01-28 14:51:39 +0100 | [diff] [blame] | 3852 | if (delta_exec > ideal_runtime) { |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 3853 | resched_curr(rq_of(cfs_rq)); |
Mike Galbraith | a9f3e2b | 2009-01-28 14:51:39 +0100 | [diff] [blame] | 3854 | /* |
| 3855 | * The current task ran long enough, ensure it doesn't get |
| 3856 | * re-elected due to buddy favours. |
| 3857 | */ |
| 3858 | clear_buddies(cfs_rq, curr); |
Mike Galbraith | f685cea | 2009-10-23 23:09:22 +0200 | [diff] [blame] | 3859 | return; |
| 3860 | } |
| 3861 | |
| 3862 | /* |
| 3863 | * Ensure that a task that missed wakeup preemption by a |
| 3864 | * narrow margin doesn't have to wait for a full slice. |
| 3865 | * This also mitigates buddy induced latencies under load. |
| 3866 | */ |
Mike Galbraith | f685cea | 2009-10-23 23:09:22 +0200 | [diff] [blame] | 3867 | if (delta_exec < sysctl_sched_min_granularity) |
| 3868 | return; |
| 3869 | |
Wang Xingchao | f4cfb33 | 2011-09-16 13:35:52 -0400 | [diff] [blame] | 3870 | se = __pick_first_entity(cfs_rq); |
| 3871 | delta = curr->vruntime - se->vruntime; |
Mike Galbraith | f685cea | 2009-10-23 23:09:22 +0200 | [diff] [blame] | 3872 | |
Wang Xingchao | f4cfb33 | 2011-09-16 13:35:52 -0400 | [diff] [blame] | 3873 | if (delta < 0) |
| 3874 | return; |
Mike Galbraith | d7d8294 | 2011-01-05 05:41:17 +0100 | [diff] [blame] | 3875 | |
Wang Xingchao | f4cfb33 | 2011-09-16 13:35:52 -0400 | [diff] [blame] | 3876 | if (delta > ideal_runtime) |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 3877 | resched_curr(rq_of(cfs_rq)); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3878 | } |
| 3879 | |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 3880 | static void |
Ingo Molnar | 8494f41 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 3881 | set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3882 | { |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 3883 | /* 'current' is not kept within the tree. */ |
| 3884 | if (se->on_rq) { |
| 3885 | /* |
| 3886 | * Any task has to be enqueued before it get to execute on |
| 3887 | * a CPU. So account for the time it spent waiting on the |
| 3888 | * runqueue. |
| 3889 | */ |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3890 | update_stats_wait_end(cfs_rq, se); |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 3891 | __dequeue_entity(cfs_rq, se); |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 3892 | update_load_avg(se, UPDATE_TG); |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 3893 | } |
| 3894 | |
Ingo Molnar | 79303e9 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 3895 | update_stats_curr_start(cfs_rq, se); |
Ingo Molnar | 429d43b | 2007-10-15 17:00:03 +0200 | [diff] [blame] | 3896 | cfs_rq->curr = se; |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3897 | |
Ingo Molnar | eba1ed4 | 2007-10-15 17:00:02 +0200 | [diff] [blame] | 3898 | /* |
| 3899 | * Track our maximum slice length, if the CPU's load is at |
| 3900 | * least twice that of our own weight (i.e. dont track it |
| 3901 | * when there are only lesser-weight tasks around): |
| 3902 | */ |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 3903 | if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) { |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3904 | schedstat_set(se->statistics.slice_max, |
| 3905 | max((u64)schedstat_val(se->statistics.slice_max), |
| 3906 | se->sum_exec_runtime - se->prev_sum_exec_runtime)); |
Ingo Molnar | eba1ed4 | 2007-10-15 17:00:02 +0200 | [diff] [blame] | 3907 | } |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3908 | |
Peter Zijlstra | 4a55b45 | 2007-09-05 14:32:49 +0200 | [diff] [blame] | 3909 | se->prev_sum_exec_runtime = se->sum_exec_runtime; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3910 | } |
| 3911 | |
Peter Zijlstra | 3f3a490 | 2008-10-24 11:06:16 +0200 | [diff] [blame] | 3912 | static int |
| 3913 | wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); |
| 3914 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3915 | /* |
| 3916 | * Pick the next process, keeping these things in mind, in this order: |
| 3917 | * 1) keep things fair between processes/task groups |
| 3918 | * 2) pick the "next" process, since someone really wants that to run |
| 3919 | * 3) pick the "last" process, for cache locality |
| 3920 | * 4) do not run the "skip" process, if something else is available |
| 3921 | */ |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3922 | static struct sched_entity * |
| 3923 | pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) |
Peter Zijlstra | aa2ac25 | 2008-03-14 21:12:12 +0100 | [diff] [blame] | 3924 | { |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3925 | struct sched_entity *left = __pick_first_entity(cfs_rq); |
| 3926 | struct sched_entity *se; |
| 3927 | |
| 3928 | /* |
| 3929 | * If curr is set we have to see if its left of the leftmost entity |
| 3930 | * still in the tree, provided there was anything in the tree at all. |
| 3931 | */ |
| 3932 | if (!left || (curr && entity_before(curr, left))) |
| 3933 | left = curr; |
| 3934 | |
| 3935 | se = left; /* ideally we run the leftmost entity */ |
Peter Zijlstra | f4b6755 | 2008-11-04 21:25:07 +0100 | [diff] [blame] | 3936 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3937 | /* |
| 3938 | * Avoid running the skip buddy, if running something else can |
| 3939 | * be done without getting too unfair. |
| 3940 | */ |
| 3941 | if (cfs_rq->skip == se) { |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3942 | struct sched_entity *second; |
| 3943 | |
| 3944 | if (se == curr) { |
| 3945 | second = __pick_first_entity(cfs_rq); |
| 3946 | } else { |
| 3947 | second = __pick_next_entity(se); |
| 3948 | if (!second || (curr && entity_before(curr, second))) |
| 3949 | second = curr; |
| 3950 | } |
| 3951 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3952 | if (second && wakeup_preempt_entity(second, left) < 1) |
| 3953 | se = second; |
| 3954 | } |
Peter Zijlstra | aa2ac25 | 2008-03-14 21:12:12 +0100 | [diff] [blame] | 3955 | |
Mike Galbraith | f685cea | 2009-10-23 23:09:22 +0200 | [diff] [blame] | 3956 | /* |
| 3957 | * Prefer last buddy, try to return the CPU to a preempted task. |
| 3958 | */ |
| 3959 | if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) |
| 3960 | se = cfs_rq->last; |
| 3961 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 3962 | /* |
| 3963 | * Someone really wants this to run. If it's not unfair, run it. |
| 3964 | */ |
| 3965 | if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) |
| 3966 | se = cfs_rq->next; |
| 3967 | |
Mike Galbraith | f685cea | 2009-10-23 23:09:22 +0200 | [diff] [blame] | 3968 | clear_buddies(cfs_rq, se); |
Peter Zijlstra | 4793241 | 2008-11-04 21:25:09 +0100 | [diff] [blame] | 3969 | |
| 3970 | return se; |
Peter Zijlstra | aa2ac25 | 2008-03-14 21:12:12 +0100 | [diff] [blame] | 3971 | } |
| 3972 | |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 3973 | static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 3974 | |
Ingo Molnar | ab6cde2 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 3975 | static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3976 | { |
| 3977 | /* |
| 3978 | * If still on the runqueue then deactivate_task() |
| 3979 | * was not called and update_curr() has to be done: |
| 3980 | */ |
| 3981 | if (prev->on_rq) |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 3982 | update_curr(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3983 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 3984 | /* throttle cfs_rqs exceeding runtime */ |
| 3985 | check_cfs_rq_runtime(cfs_rq); |
| 3986 | |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3987 | check_spread(cfs_rq, prev); |
Mel Gorman | cb25176 | 2016-02-05 09:08:36 +0000 | [diff] [blame] | 3988 | |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 3989 | if (prev->on_rq) { |
Josh Poimboeuf | 4fa8d29 | 2016-06-17 12:43:26 -0500 | [diff] [blame] | 3990 | update_stats_wait_start(cfs_rq, prev); |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 3991 | /* Put 'current' back into the tree. */ |
| 3992 | __enqueue_entity(cfs_rq, prev); |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 3993 | /* in !on_rq case, update occurred at dequeue */ |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 3994 | update_load_avg(prev, 0); |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 3995 | } |
Ingo Molnar | 429d43b | 2007-10-15 17:00:03 +0200 | [diff] [blame] | 3996 | cfs_rq->curr = NULL; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 3997 | } |
| 3998 | |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 3999 | static void |
| 4000 | entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4001 | { |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4002 | /* |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 4003 | * Update run-time statistics of the 'current'. |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4004 | */ |
Dmitry Adamushko | 30cfdcf | 2007-10-15 17:00:07 +0200 | [diff] [blame] | 4005 | update_curr(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4006 | |
Paul Turner | 43365bd | 2010-12-15 19:10:17 -0800 | [diff] [blame] | 4007 | /* |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 4008 | * Ensure that runnable average is periodically updated. |
| 4009 | */ |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 4010 | update_load_avg(curr, UPDATE_TG); |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 4011 | update_cfs_shares(curr); |
Paul Turner | 9d85f21 | 2012-10-04 13:18:29 +0200 | [diff] [blame] | 4012 | |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4013 | #ifdef CONFIG_SCHED_HRTICK |
| 4014 | /* |
| 4015 | * queued ticks are scheduled to match the slice, so don't bother |
| 4016 | * validating it and just reschedule. |
| 4017 | */ |
Harvey Harrison | 983ed7a | 2008-04-24 18:17:55 -0700 | [diff] [blame] | 4018 | if (queued) { |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 4019 | resched_curr(rq_of(cfs_rq)); |
Harvey Harrison | 983ed7a | 2008-04-24 18:17:55 -0700 | [diff] [blame] | 4020 | return; |
| 4021 | } |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4022 | /* |
| 4023 | * don't let the period tick interfere with the hrtick preemption |
| 4024 | */ |
| 4025 | if (!sched_feat(DOUBLE_TICK) && |
| 4026 | hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) |
| 4027 | return; |
| 4028 | #endif |
| 4029 | |
Yong Zhang | 2c2efae | 2011-07-29 16:20:33 +0800 | [diff] [blame] | 4030 | if (cfs_rq->nr_running > 1) |
Ingo Molnar | 2e09bf5 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 4031 | check_preempt_tick(cfs_rq, curr); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4032 | } |
| 4033 | |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 4034 | |
| 4035 | /************************************************** |
| 4036 | * CFS bandwidth control machinery |
| 4037 | */ |
| 4038 | |
| 4039 | #ifdef CONFIG_CFS_BANDWIDTH |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4040 | |
| 4041 | #ifdef HAVE_JUMP_LABEL |
Ingo Molnar | c5905af | 2012-02-24 08:31:31 +0100 | [diff] [blame] | 4042 | static struct static_key __cfs_bandwidth_used; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4043 | |
| 4044 | static inline bool cfs_bandwidth_used(void) |
| 4045 | { |
Ingo Molnar | c5905af | 2012-02-24 08:31:31 +0100 | [diff] [blame] | 4046 | return static_key_false(&__cfs_bandwidth_used); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4047 | } |
| 4048 | |
Ben Segall | 1ee14e6 | 2013-10-16 11:16:12 -0700 | [diff] [blame] | 4049 | void cfs_bandwidth_usage_inc(void) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4050 | { |
Ben Segall | 1ee14e6 | 2013-10-16 11:16:12 -0700 | [diff] [blame] | 4051 | static_key_slow_inc(&__cfs_bandwidth_used); |
| 4052 | } |
| 4053 | |
| 4054 | void cfs_bandwidth_usage_dec(void) |
| 4055 | { |
| 4056 | static_key_slow_dec(&__cfs_bandwidth_used); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4057 | } |
| 4058 | #else /* HAVE_JUMP_LABEL */ |
| 4059 | static bool cfs_bandwidth_used(void) |
| 4060 | { |
| 4061 | return true; |
| 4062 | } |
| 4063 | |
Ben Segall | 1ee14e6 | 2013-10-16 11:16:12 -0700 | [diff] [blame] | 4064 | void cfs_bandwidth_usage_inc(void) {} |
| 4065 | void cfs_bandwidth_usage_dec(void) {} |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4066 | #endif /* HAVE_JUMP_LABEL */ |
| 4067 | |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 4068 | /* |
| 4069 | * default period for cfs group bandwidth. |
| 4070 | * default: 0.1s, units: nanoseconds |
| 4071 | */ |
| 4072 | static inline u64 default_cfs_period(void) |
| 4073 | { |
| 4074 | return 100000000ULL; |
| 4075 | } |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4076 | |
| 4077 | static inline u64 sched_cfs_bandwidth_slice(void) |
| 4078 | { |
| 4079 | return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; |
| 4080 | } |
| 4081 | |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4082 | /* |
| 4083 | * Replenish runtime according to assigned quota and update expiration time. |
| 4084 | * We use sched_clock_cpu directly instead of rq->clock to avoid adding |
| 4085 | * additional synchronization around rq->lock. |
| 4086 | * |
| 4087 | * requires cfs_b->lock |
| 4088 | */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4089 | void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4090 | { |
| 4091 | u64 now; |
| 4092 | |
| 4093 | if (cfs_b->quota == RUNTIME_INF) |
| 4094 | return; |
| 4095 | |
| 4096 | now = sched_clock_cpu(smp_processor_id()); |
| 4097 | cfs_b->runtime = cfs_b->quota; |
| 4098 | cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); |
| 4099 | } |
| 4100 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4101 | static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) |
| 4102 | { |
| 4103 | return &tg->cfs_bandwidth; |
| 4104 | } |
| 4105 | |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4106 | /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ |
| 4107 | static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) |
| 4108 | { |
| 4109 | if (unlikely(cfs_rq->throttle_count)) |
Xunlei Pang | 1a99ae3 | 2016-05-10 21:03:18 +0800 | [diff] [blame] | 4110 | return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time; |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4111 | |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 4112 | return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4113 | } |
| 4114 | |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4115 | /* returns 0 on failure to allocate runtime */ |
| 4116 | static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4117 | { |
| 4118 | struct task_group *tg = cfs_rq->tg; |
| 4119 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4120 | u64 amount = 0, min_amount, expires; |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4121 | |
| 4122 | /* note: this is a positive sum as runtime_remaining <= 0 */ |
| 4123 | min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining; |
| 4124 | |
| 4125 | raw_spin_lock(&cfs_b->lock); |
| 4126 | if (cfs_b->quota == RUNTIME_INF) |
| 4127 | amount = min_amount; |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4128 | else { |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 4129 | start_cfs_bandwidth(cfs_b); |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4130 | |
| 4131 | if (cfs_b->runtime > 0) { |
| 4132 | amount = min(cfs_b->runtime, min_amount); |
| 4133 | cfs_b->runtime -= amount; |
| 4134 | cfs_b->idle = 0; |
| 4135 | } |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4136 | } |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4137 | expires = cfs_b->runtime_expires; |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4138 | raw_spin_unlock(&cfs_b->lock); |
| 4139 | |
| 4140 | cfs_rq->runtime_remaining += amount; |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4141 | /* |
| 4142 | * we may have advanced our local expiration to account for allowed |
| 4143 | * spread between our sched_clock and the one on which runtime was |
| 4144 | * issued. |
| 4145 | */ |
| 4146 | if ((s64)(expires - cfs_rq->runtime_expires) > 0) |
| 4147 | cfs_rq->runtime_expires = expires; |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4148 | |
| 4149 | return cfs_rq->runtime_remaining > 0; |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4150 | } |
| 4151 | |
| 4152 | /* |
| 4153 | * Note: This depends on the synchronization provided by sched_clock and the |
| 4154 | * fact that rq->clock snapshots this value. |
| 4155 | */ |
| 4156 | static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
| 4157 | { |
| 4158 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4159 | |
| 4160 | /* if the deadline is ahead of our clock, nothing to do */ |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 4161 | if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0)) |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4162 | return; |
| 4163 | |
| 4164 | if (cfs_rq->runtime_remaining < 0) |
| 4165 | return; |
| 4166 | |
| 4167 | /* |
| 4168 | * If the local deadline has passed we have to consider the |
| 4169 | * possibility that our sched_clock is 'fast' and the global deadline |
| 4170 | * has not truly expired. |
| 4171 | * |
| 4172 | * Fortunately we can check determine whether this the case by checking |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4173 | * whether the global deadline has advanced. It is valid to compare |
| 4174 | * cfs_b->runtime_expires without any locks since we only care about |
| 4175 | * exact equality, so a partial write will still work. |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4176 | */ |
| 4177 | |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4178 | if (cfs_rq->runtime_expires != cfs_b->runtime_expires) { |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4179 | /* extend local deadline, drift is bounded above by 2 ticks */ |
| 4180 | cfs_rq->runtime_expires += TICK_NSEC; |
| 4181 | } else { |
| 4182 | /* global deadline is ahead, expiration has passed */ |
| 4183 | cfs_rq->runtime_remaining = 0; |
| 4184 | } |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4185 | } |
| 4186 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 4187 | static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4188 | { |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4189 | /* dock delta_exec before expiring quota (as it could span periods) */ |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4190 | cfs_rq->runtime_remaining -= delta_exec; |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4191 | expire_cfs_rq_runtime(cfs_rq); |
| 4192 | |
| 4193 | if (likely(cfs_rq->runtime_remaining > 0)) |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4194 | return; |
| 4195 | |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4196 | /* |
| 4197 | * if we're unable to extend our runtime we resched so that the active |
| 4198 | * hierarchy can be throttled |
| 4199 | */ |
| 4200 | if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 4201 | resched_curr(rq_of(cfs_rq)); |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4202 | } |
| 4203 | |
Peter Zijlstra | 6c16a6d | 2012-03-21 13:07:16 -0700 | [diff] [blame] | 4204 | static __always_inline |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 4205 | void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4206 | { |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 4207 | if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 4208 | return; |
| 4209 | |
| 4210 | __account_cfs_rq_runtime(cfs_rq, delta_exec); |
| 4211 | } |
| 4212 | |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4213 | static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) |
| 4214 | { |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 4215 | return cfs_bandwidth_used() && cfs_rq->throttled; |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4216 | } |
| 4217 | |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4218 | /* check whether cfs_rq, or any parent, is throttled */ |
| 4219 | static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) |
| 4220 | { |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 4221 | return cfs_bandwidth_used() && cfs_rq->throttle_count; |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4222 | } |
| 4223 | |
| 4224 | /* |
| 4225 | * Ensure that neither of the group entities corresponding to src_cpu or |
| 4226 | * dest_cpu are members of a throttled hierarchy when performing group |
| 4227 | * load-balance operations. |
| 4228 | */ |
| 4229 | static inline int throttled_lb_pair(struct task_group *tg, |
| 4230 | int src_cpu, int dest_cpu) |
| 4231 | { |
| 4232 | struct cfs_rq *src_cfs_rq, *dest_cfs_rq; |
| 4233 | |
| 4234 | src_cfs_rq = tg->cfs_rq[src_cpu]; |
| 4235 | dest_cfs_rq = tg->cfs_rq[dest_cpu]; |
| 4236 | |
| 4237 | return throttled_hierarchy(src_cfs_rq) || |
| 4238 | throttled_hierarchy(dest_cfs_rq); |
| 4239 | } |
| 4240 | |
| 4241 | /* updated child weight may affect parent so we have to do this bottom up */ |
| 4242 | static int tg_unthrottle_up(struct task_group *tg, void *data) |
| 4243 | { |
| 4244 | struct rq *rq = data; |
| 4245 | struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; |
| 4246 | |
| 4247 | cfs_rq->throttle_count--; |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4248 | if (!cfs_rq->throttle_count) { |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4249 | /* adjust cfs_rq_clock_task() */ |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 4250 | cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4251 | cfs_rq->throttled_clock_task; |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4252 | } |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4253 | |
| 4254 | return 0; |
| 4255 | } |
| 4256 | |
| 4257 | static int tg_throttle_down(struct task_group *tg, void *data) |
| 4258 | { |
| 4259 | struct rq *rq = data; |
| 4260 | struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; |
| 4261 | |
Paul Turner | 8295836 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4262 | /* group is entering throttled state, stop time */ |
| 4263 | if (!cfs_rq->throttle_count) |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 4264 | cfs_rq->throttled_clock_task = rq_clock_task(rq); |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4265 | cfs_rq->throttle_count++; |
| 4266 | |
| 4267 | return 0; |
| 4268 | } |
| 4269 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4270 | static void throttle_cfs_rq(struct cfs_rq *cfs_rq) |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4271 | { |
| 4272 | struct rq *rq = rq_of(cfs_rq); |
| 4273 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); |
| 4274 | struct sched_entity *se; |
| 4275 | long task_delta, dequeue = 1; |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 4276 | bool empty; |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4277 | |
| 4278 | se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; |
| 4279 | |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4280 | /* freeze hierarchy runnable averages while throttled */ |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4281 | rcu_read_lock(); |
| 4282 | walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); |
| 4283 | rcu_read_unlock(); |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4284 | |
| 4285 | task_delta = cfs_rq->h_nr_running; |
| 4286 | for_each_sched_entity(se) { |
| 4287 | struct cfs_rq *qcfs_rq = cfs_rq_of(se); |
| 4288 | /* throttled entity or throttle-on-deactivate */ |
| 4289 | if (!se->on_rq) |
| 4290 | break; |
| 4291 | |
| 4292 | if (dequeue) |
| 4293 | dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); |
| 4294 | qcfs_rq->h_nr_running -= task_delta; |
| 4295 | |
| 4296 | if (qcfs_rq->load.weight) |
| 4297 | dequeue = 0; |
| 4298 | } |
| 4299 | |
| 4300 | if (!se) |
Kirill Tkhai | 7246544 | 2014-05-09 03:00:14 +0400 | [diff] [blame] | 4301 | sub_nr_running(rq, task_delta); |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4302 | |
| 4303 | cfs_rq->throttled = 1; |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 4304 | cfs_rq->throttled_clock = rq_clock(rq); |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4305 | raw_spin_lock(&cfs_b->lock); |
Cong Wang | d49db34 | 2015-06-24 12:41:47 -0700 | [diff] [blame] | 4306 | empty = list_empty(&cfs_b->throttled_cfs_rq); |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 4307 | |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4308 | /* |
| 4309 | * Add to the _head_ of the list, so that an already-started |
| 4310 | * distribute_cfs_runtime will not see us |
| 4311 | */ |
| 4312 | list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 4313 | |
| 4314 | /* |
| 4315 | * If we're the first throttled task, make sure the bandwidth |
| 4316 | * timer is running. |
| 4317 | */ |
| 4318 | if (empty) |
| 4319 | start_cfs_bandwidth(cfs_b); |
| 4320 | |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4321 | raw_spin_unlock(&cfs_b->lock); |
| 4322 | } |
| 4323 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4324 | void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4325 | { |
| 4326 | struct rq *rq = rq_of(cfs_rq); |
| 4327 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); |
| 4328 | struct sched_entity *se; |
| 4329 | int enqueue = 1; |
| 4330 | long task_delta; |
| 4331 | |
Michael Wang | 22b958d | 2013-06-04 14:23:39 +0800 | [diff] [blame] | 4332 | se = cfs_rq->tg->se[cpu_of(rq)]; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4333 | |
| 4334 | cfs_rq->throttled = 0; |
Frederic Weisbecker | 1a55af2 | 2013-04-12 01:51:01 +0200 | [diff] [blame] | 4335 | |
| 4336 | update_rq_clock(rq); |
| 4337 | |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4338 | raw_spin_lock(&cfs_b->lock); |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 4339 | cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4340 | list_del_rcu(&cfs_rq->throttled_list); |
| 4341 | raw_spin_unlock(&cfs_b->lock); |
| 4342 | |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4343 | /* update hierarchical throttle state */ |
| 4344 | walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); |
| 4345 | |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4346 | if (!cfs_rq->load.weight) |
| 4347 | return; |
| 4348 | |
| 4349 | task_delta = cfs_rq->h_nr_running; |
| 4350 | for_each_sched_entity(se) { |
| 4351 | if (se->on_rq) |
| 4352 | enqueue = 0; |
| 4353 | |
| 4354 | cfs_rq = cfs_rq_of(se); |
| 4355 | if (enqueue) |
| 4356 | enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); |
| 4357 | cfs_rq->h_nr_running += task_delta; |
| 4358 | |
| 4359 | if (cfs_rq_throttled(cfs_rq)) |
| 4360 | break; |
| 4361 | } |
| 4362 | |
| 4363 | if (!se) |
Kirill Tkhai | 7246544 | 2014-05-09 03:00:14 +0400 | [diff] [blame] | 4364 | add_nr_running(rq, task_delta); |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4365 | |
| 4366 | /* determine whether we need to wake up potentially idle cpu */ |
| 4367 | if (rq->curr == rq->idle && rq->cfs.nr_running) |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 4368 | resched_curr(rq); |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4369 | } |
| 4370 | |
| 4371 | static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, |
| 4372 | u64 remaining, u64 expires) |
| 4373 | { |
| 4374 | struct cfs_rq *cfs_rq; |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4375 | u64 runtime; |
| 4376 | u64 starting_runtime = remaining; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4377 | |
| 4378 | rcu_read_lock(); |
| 4379 | list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, |
| 4380 | throttled_list) { |
| 4381 | struct rq *rq = rq_of(cfs_rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 4382 | struct rq_flags rf; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4383 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 4384 | rq_lock(rq, &rf); |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4385 | if (!cfs_rq_throttled(cfs_rq)) |
| 4386 | goto next; |
| 4387 | |
| 4388 | runtime = -cfs_rq->runtime_remaining + 1; |
| 4389 | if (runtime > remaining) |
| 4390 | runtime = remaining; |
| 4391 | remaining -= runtime; |
| 4392 | |
| 4393 | cfs_rq->runtime_remaining += runtime; |
| 4394 | cfs_rq->runtime_expires = expires; |
| 4395 | |
| 4396 | /* we check whether we're throttled above */ |
| 4397 | if (cfs_rq->runtime_remaining > 0) |
| 4398 | unthrottle_cfs_rq(cfs_rq); |
| 4399 | |
| 4400 | next: |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 4401 | rq_unlock(rq, &rf); |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4402 | |
| 4403 | if (!remaining) |
| 4404 | break; |
| 4405 | } |
| 4406 | rcu_read_unlock(); |
| 4407 | |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4408 | return starting_runtime - remaining; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4409 | } |
| 4410 | |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4411 | /* |
| 4412 | * Responsible for refilling a task_group's bandwidth and unthrottling its |
| 4413 | * cfs_rqs as appropriate. If there has been no activity within the last |
| 4414 | * period the timer is deactivated until scheduling resumes; cfs_b->idle is |
| 4415 | * used to track this state. |
| 4416 | */ |
| 4417 | static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) |
| 4418 | { |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4419 | u64 runtime, runtime_expires; |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4420 | int throttled; |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4421 | |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4422 | /* no need to continue the timer with no bandwidth constraint */ |
| 4423 | if (cfs_b->quota == RUNTIME_INF) |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4424 | goto out_deactivate; |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4425 | |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4426 | throttled = !list_empty(&cfs_b->throttled_cfs_rq); |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 4427 | cfs_b->nr_periods += overrun; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4428 | |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4429 | /* |
| 4430 | * idle depends on !throttled (for the case of a large deficit), and if |
| 4431 | * we're going inactive then everything else can be deferred |
| 4432 | */ |
| 4433 | if (cfs_b->idle && !throttled) |
| 4434 | goto out_deactivate; |
Paul Turner | a9cf55b | 2011-07-21 09:43:32 -0700 | [diff] [blame] | 4435 | |
| 4436 | __refill_cfs_bandwidth_runtime(cfs_b); |
| 4437 | |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4438 | if (!throttled) { |
| 4439 | /* mark as potentially idle for the upcoming period */ |
| 4440 | cfs_b->idle = 1; |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4441 | return 0; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4442 | } |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4443 | |
Nikhil Rao | e8da1b1 | 2011-07-21 09:43:40 -0700 | [diff] [blame] | 4444 | /* account preceding periods in which throttling occurred */ |
| 4445 | cfs_b->nr_throttled += overrun; |
| 4446 | |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4447 | runtime_expires = cfs_b->runtime_expires; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4448 | |
| 4449 | /* |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4450 | * This check is repeated as we are holding onto the new bandwidth while |
| 4451 | * we unthrottle. This can potentially race with an unthrottled group |
| 4452 | * trying to acquire new bandwidth from the global pool. This can result |
| 4453 | * in us over-using our runtime if it is all used during this loop, but |
| 4454 | * only by limited amounts in that extreme case. |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4455 | */ |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4456 | while (throttled && cfs_b->runtime > 0) { |
| 4457 | runtime = cfs_b->runtime; |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4458 | raw_spin_unlock(&cfs_b->lock); |
| 4459 | /* we can't nest cfs_b->lock while distributing bandwidth */ |
| 4460 | runtime = distribute_cfs_runtime(cfs_b, runtime, |
| 4461 | runtime_expires); |
| 4462 | raw_spin_lock(&cfs_b->lock); |
| 4463 | |
| 4464 | throttled = !list_empty(&cfs_b->throttled_cfs_rq); |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4465 | |
| 4466 | cfs_b->runtime -= min(runtime, cfs_b->runtime); |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4467 | } |
| 4468 | |
Paul Turner | 671fd9d | 2011-07-21 09:43:34 -0700 | [diff] [blame] | 4469 | /* |
| 4470 | * While we are ensured activity in the period following an |
| 4471 | * unthrottle, this also covers the case in which the new bandwidth is |
| 4472 | * insufficient to cover the existing bandwidth deficit. (Forcing the |
| 4473 | * timer to remain active while there are any throttled entities.) |
| 4474 | */ |
| 4475 | cfs_b->idle = 0; |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4476 | |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4477 | return 0; |
| 4478 | |
| 4479 | out_deactivate: |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4480 | return 1; |
Paul Turner | 58088ad | 2011-07-21 09:43:31 -0700 | [diff] [blame] | 4481 | } |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4482 | |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4483 | /* a cfs_rq won't donate quota below this amount */ |
| 4484 | static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; |
| 4485 | /* minimum remaining period time to redistribute slack quota */ |
| 4486 | static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; |
| 4487 | /* how long we wait to gather additional slack before distributing */ |
| 4488 | static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; |
| 4489 | |
Ben Segall | db06e78 | 2013-10-16 11:16:17 -0700 | [diff] [blame] | 4490 | /* |
| 4491 | * Are we near the end of the current quota period? |
| 4492 | * |
| 4493 | * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the |
Thomas Gleixner | 4961b6e | 2015-04-14 21:09:05 +0000 | [diff] [blame] | 4494 | * hrtimer base being cleared by hrtimer_start. In the case of |
Ben Segall | db06e78 | 2013-10-16 11:16:17 -0700 | [diff] [blame] | 4495 | * migrate_hrtimers, base is never cleared, so we are fine. |
| 4496 | */ |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4497 | static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) |
| 4498 | { |
| 4499 | struct hrtimer *refresh_timer = &cfs_b->period_timer; |
| 4500 | u64 remaining; |
| 4501 | |
| 4502 | /* if the call-back is running a quota refresh is already occurring */ |
| 4503 | if (hrtimer_callback_running(refresh_timer)) |
| 4504 | return 1; |
| 4505 | |
| 4506 | /* is a quota refresh about to occur? */ |
| 4507 | remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); |
| 4508 | if (remaining < min_expire) |
| 4509 | return 1; |
| 4510 | |
| 4511 | return 0; |
| 4512 | } |
| 4513 | |
| 4514 | static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) |
| 4515 | { |
| 4516 | u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; |
| 4517 | |
| 4518 | /* if there's a quota refresh soon don't bother with slack */ |
| 4519 | if (runtime_refresh_within(cfs_b, min_left)) |
| 4520 | return; |
| 4521 | |
Peter Zijlstra | 4cfafd3 | 2015-05-14 12:23:11 +0200 | [diff] [blame] | 4522 | hrtimer_start(&cfs_b->slack_timer, |
| 4523 | ns_to_ktime(cfs_bandwidth_slack_period), |
| 4524 | HRTIMER_MODE_REL); |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4525 | } |
| 4526 | |
| 4527 | /* we know any runtime found here is valid as update_curr() precedes return */ |
| 4528 | static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
| 4529 | { |
| 4530 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); |
| 4531 | s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; |
| 4532 | |
| 4533 | if (slack_runtime <= 0) |
| 4534 | return; |
| 4535 | |
| 4536 | raw_spin_lock(&cfs_b->lock); |
| 4537 | if (cfs_b->quota != RUNTIME_INF && |
| 4538 | cfs_rq->runtime_expires == cfs_b->runtime_expires) { |
| 4539 | cfs_b->runtime += slack_runtime; |
| 4540 | |
| 4541 | /* we are under rq->lock, defer unthrottling using a timer */ |
| 4542 | if (cfs_b->runtime > sched_cfs_bandwidth_slice() && |
| 4543 | !list_empty(&cfs_b->throttled_cfs_rq)) |
| 4544 | start_cfs_slack_bandwidth(cfs_b); |
| 4545 | } |
| 4546 | raw_spin_unlock(&cfs_b->lock); |
| 4547 | |
| 4548 | /* even if it's not valid for return we don't want to try again */ |
| 4549 | cfs_rq->runtime_remaining -= slack_runtime; |
| 4550 | } |
| 4551 | |
| 4552 | static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
| 4553 | { |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 4554 | if (!cfs_bandwidth_used()) |
| 4555 | return; |
| 4556 | |
Paul Turner | fccfdc6 | 2011-11-07 20:26:34 -0800 | [diff] [blame] | 4557 | if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4558 | return; |
| 4559 | |
| 4560 | __return_cfs_rq_runtime(cfs_rq); |
| 4561 | } |
| 4562 | |
| 4563 | /* |
| 4564 | * This is done with a timer (instead of inline with bandwidth return) since |
| 4565 | * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. |
| 4566 | */ |
| 4567 | static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) |
| 4568 | { |
| 4569 | u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); |
| 4570 | u64 expires; |
| 4571 | |
| 4572 | /* confirm we're still not at a refresh boundary */ |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4573 | raw_spin_lock(&cfs_b->lock); |
Ben Segall | db06e78 | 2013-10-16 11:16:17 -0700 | [diff] [blame] | 4574 | if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { |
| 4575 | raw_spin_unlock(&cfs_b->lock); |
| 4576 | return; |
| 4577 | } |
| 4578 | |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4579 | if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4580 | runtime = cfs_b->runtime; |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4581 | |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4582 | expires = cfs_b->runtime_expires; |
| 4583 | raw_spin_unlock(&cfs_b->lock); |
| 4584 | |
| 4585 | if (!runtime) |
| 4586 | return; |
| 4587 | |
| 4588 | runtime = distribute_cfs_runtime(cfs_b, runtime, expires); |
| 4589 | |
| 4590 | raw_spin_lock(&cfs_b->lock); |
| 4591 | if (expires == cfs_b->runtime_expires) |
Ben Segall | c06f04c | 2014-06-20 15:21:20 -0700 | [diff] [blame] | 4592 | cfs_b->runtime -= min(runtime, cfs_b->runtime); |
Paul Turner | d8b4986 | 2011-07-21 09:43:41 -0700 | [diff] [blame] | 4593 | raw_spin_unlock(&cfs_b->lock); |
| 4594 | } |
| 4595 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4596 | /* |
| 4597 | * When a group wakes up we want to make sure that its quota is not already |
| 4598 | * expired/exceeded, otherwise it may be allowed to steal additional ticks of |
| 4599 | * runtime as update_curr() throttling can not not trigger until it's on-rq. |
| 4600 | */ |
| 4601 | static void check_enqueue_throttle(struct cfs_rq *cfs_rq) |
| 4602 | { |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 4603 | if (!cfs_bandwidth_used()) |
| 4604 | return; |
| 4605 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4606 | /* an active group must be handled by the update_curr()->put() path */ |
| 4607 | if (!cfs_rq->runtime_enabled || cfs_rq->curr) |
| 4608 | return; |
| 4609 | |
| 4610 | /* ensure the group is not already throttled */ |
| 4611 | if (cfs_rq_throttled(cfs_rq)) |
| 4612 | return; |
| 4613 | |
| 4614 | /* update runtime allocation */ |
| 4615 | account_cfs_rq_runtime(cfs_rq, 0); |
| 4616 | if (cfs_rq->runtime_remaining <= 0) |
| 4617 | throttle_cfs_rq(cfs_rq); |
| 4618 | } |
| 4619 | |
Peter Zijlstra | 55e16d3 | 2016-06-22 15:14:26 +0200 | [diff] [blame] | 4620 | static void sync_throttle(struct task_group *tg, int cpu) |
| 4621 | { |
| 4622 | struct cfs_rq *pcfs_rq, *cfs_rq; |
| 4623 | |
| 4624 | if (!cfs_bandwidth_used()) |
| 4625 | return; |
| 4626 | |
| 4627 | if (!tg->parent) |
| 4628 | return; |
| 4629 | |
| 4630 | cfs_rq = tg->cfs_rq[cpu]; |
| 4631 | pcfs_rq = tg->parent->cfs_rq[cpu]; |
| 4632 | |
| 4633 | cfs_rq->throttle_count = pcfs_rq->throttle_count; |
Xunlei Pang | b892212 | 2016-07-09 15:54:22 +0800 | [diff] [blame] | 4634 | cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu)); |
Peter Zijlstra | 55e16d3 | 2016-06-22 15:14:26 +0200 | [diff] [blame] | 4635 | } |
| 4636 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4637 | /* conditionally throttle active cfs_rq's from put_prev_entity() */ |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 4638 | static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4639 | { |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 4640 | if (!cfs_bandwidth_used()) |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 4641 | return false; |
Paul Turner | 56f570e | 2011-11-07 20:26:33 -0800 | [diff] [blame] | 4642 | |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4643 | if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 4644 | return false; |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4645 | |
| 4646 | /* |
| 4647 | * it's possible for a throttled entity to be forced into a running |
| 4648 | * state (e.g. set_curr_task), in this case we're finished. |
| 4649 | */ |
| 4650 | if (cfs_rq_throttled(cfs_rq)) |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 4651 | return true; |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4652 | |
| 4653 | throttle_cfs_rq(cfs_rq); |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 4654 | return true; |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4655 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4656 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4657 | static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) |
| 4658 | { |
| 4659 | struct cfs_bandwidth *cfs_b = |
| 4660 | container_of(timer, struct cfs_bandwidth, slack_timer); |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 4661 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4662 | do_sched_cfs_slack_timer(cfs_b); |
| 4663 | |
| 4664 | return HRTIMER_NORESTART; |
| 4665 | } |
| 4666 | |
| 4667 | static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) |
| 4668 | { |
| 4669 | struct cfs_bandwidth *cfs_b = |
| 4670 | container_of(timer, struct cfs_bandwidth, period_timer); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4671 | int overrun; |
| 4672 | int idle = 0; |
| 4673 | |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4674 | raw_spin_lock(&cfs_b->lock); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4675 | for (;;) { |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 4676 | overrun = hrtimer_forward_now(timer, cfs_b->period); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4677 | if (!overrun) |
| 4678 | break; |
| 4679 | |
| 4680 | idle = do_sched_cfs_period_timer(cfs_b, overrun); |
| 4681 | } |
Peter Zijlstra | 4cfafd3 | 2015-05-14 12:23:11 +0200 | [diff] [blame] | 4682 | if (idle) |
| 4683 | cfs_b->period_active = 0; |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4684 | raw_spin_unlock(&cfs_b->lock); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4685 | |
| 4686 | return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; |
| 4687 | } |
| 4688 | |
| 4689 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) |
| 4690 | { |
| 4691 | raw_spin_lock_init(&cfs_b->lock); |
| 4692 | cfs_b->runtime = 0; |
| 4693 | cfs_b->quota = RUNTIME_INF; |
| 4694 | cfs_b->period = ns_to_ktime(default_cfs_period()); |
| 4695 | |
| 4696 | INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); |
Peter Zijlstra | 4cfafd3 | 2015-05-14 12:23:11 +0200 | [diff] [blame] | 4697 | hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4698 | cfs_b->period_timer.function = sched_cfs_period_timer; |
| 4699 | hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 4700 | cfs_b->slack_timer.function = sched_cfs_slack_timer; |
| 4701 | } |
| 4702 | |
| 4703 | static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
| 4704 | { |
| 4705 | cfs_rq->runtime_enabled = 0; |
| 4706 | INIT_LIST_HEAD(&cfs_rq->throttled_list); |
| 4707 | } |
| 4708 | |
Peter Zijlstra | 77a4d1a | 2015-04-15 11:41:57 +0200 | [diff] [blame] | 4709 | void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4710 | { |
Peter Zijlstra | 4cfafd3 | 2015-05-14 12:23:11 +0200 | [diff] [blame] | 4711 | lockdep_assert_held(&cfs_b->lock); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4712 | |
Peter Zijlstra | 4cfafd3 | 2015-05-14 12:23:11 +0200 | [diff] [blame] | 4713 | if (!cfs_b->period_active) { |
| 4714 | cfs_b->period_active = 1; |
| 4715 | hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); |
| 4716 | hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); |
| 4717 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4718 | } |
| 4719 | |
| 4720 | static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) |
| 4721 | { |
Tetsuo Handa | 7f1a169 | 2014-12-25 15:51:21 +0900 | [diff] [blame] | 4722 | /* init_cfs_bandwidth() was not called */ |
| 4723 | if (!cfs_b->throttled_cfs_rq.next) |
| 4724 | return; |
| 4725 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4726 | hrtimer_cancel(&cfs_b->period_timer); |
| 4727 | hrtimer_cancel(&cfs_b->slack_timer); |
| 4728 | } |
| 4729 | |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4730 | /* |
| 4731 | * Both these cpu hotplug callbacks race against unregister_fair_sched_group() |
| 4732 | * |
| 4733 | * The race is harmless, since modifying bandwidth settings of unhooked group |
| 4734 | * bits doesn't do much. |
| 4735 | */ |
| 4736 | |
| 4737 | /* cpu online calback */ |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 4738 | static void __maybe_unused update_runtime_enabled(struct rq *rq) |
| 4739 | { |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4740 | struct task_group *tg; |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 4741 | |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4742 | lockdep_assert_held(&rq->lock); |
| 4743 | |
| 4744 | rcu_read_lock(); |
| 4745 | list_for_each_entry_rcu(tg, &task_groups, list) { |
| 4746 | struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; |
| 4747 | struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 4748 | |
| 4749 | raw_spin_lock(&cfs_b->lock); |
| 4750 | cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; |
| 4751 | raw_spin_unlock(&cfs_b->lock); |
| 4752 | } |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4753 | rcu_read_unlock(); |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 4754 | } |
| 4755 | |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4756 | /* cpu offline callback */ |
Arnd Bergmann | 38dc334 | 2013-01-25 14:14:22 +0000 | [diff] [blame] | 4757 | static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4758 | { |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4759 | struct task_group *tg; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4760 | |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4761 | lockdep_assert_held(&rq->lock); |
| 4762 | |
| 4763 | rcu_read_lock(); |
| 4764 | list_for_each_entry_rcu(tg, &task_groups, list) { |
| 4765 | struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; |
| 4766 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4767 | if (!cfs_rq->runtime_enabled) |
| 4768 | continue; |
| 4769 | |
| 4770 | /* |
| 4771 | * clock_task is not advancing so we just need to make sure |
| 4772 | * there's some valid quota amount |
| 4773 | */ |
Ben Segall | 51f2176 | 2014-05-19 15:49:45 -0700 | [diff] [blame] | 4774 | cfs_rq->runtime_remaining = 1; |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 4775 | /* |
| 4776 | * Offline rq is schedulable till cpu is completely disabled |
| 4777 | * in take_cpu_down(), so we prevent new cfs throttling here. |
| 4778 | */ |
| 4779 | cfs_rq->runtime_enabled = 0; |
| 4780 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4781 | if (cfs_rq_throttled(cfs_rq)) |
| 4782 | unthrottle_cfs_rq(cfs_rq); |
| 4783 | } |
Peter Zijlstra | 502ce00 | 2017-05-04 15:31:22 +0200 | [diff] [blame] | 4784 | rcu_read_unlock(); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4785 | } |
| 4786 | |
| 4787 | #else /* CONFIG_CFS_BANDWIDTH */ |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4788 | static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) |
| 4789 | { |
Frederic Weisbecker | 78becc2 | 2013-04-12 01:51:02 +0200 | [diff] [blame] | 4790 | return rq_clock_task(rq_of(cfs_rq)); |
Paul Turner | f1b1728 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 4791 | } |
| 4792 | |
Peter Zijlstra | 9dbdb15 | 2013-11-18 18:27:06 +0100 | [diff] [blame] | 4793 | static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 4794 | static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } |
Paul Turner | d3d9dc3 | 2011-07-21 09:43:39 -0700 | [diff] [blame] | 4795 | static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} |
Peter Zijlstra | 55e16d3 | 2016-06-22 15:14:26 +0200 | [diff] [blame] | 4796 | static inline void sync_throttle(struct task_group *tg, int cpu) {} |
Peter Zijlstra | 6c16a6d | 2012-03-21 13:07:16 -0700 | [diff] [blame] | 4797 | static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4798 | |
| 4799 | static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) |
| 4800 | { |
| 4801 | return 0; |
| 4802 | } |
Paul Turner | 64660c8 | 2011-07-21 09:43:36 -0700 | [diff] [blame] | 4803 | |
| 4804 | static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) |
| 4805 | { |
| 4806 | return 0; |
| 4807 | } |
| 4808 | |
| 4809 | static inline int throttled_lb_pair(struct task_group *tg, |
| 4810 | int src_cpu, int dest_cpu) |
| 4811 | { |
| 4812 | return 0; |
| 4813 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4814 | |
| 4815 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} |
| 4816 | |
| 4817 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 4818 | static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} |
Paul Turner | ab84d31 | 2011-07-21 09:43:28 -0700 | [diff] [blame] | 4819 | #endif |
| 4820 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4821 | static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) |
| 4822 | { |
| 4823 | return NULL; |
| 4824 | } |
| 4825 | static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 4826 | static inline void update_runtime_enabled(struct rq *rq) {} |
Peter Boonstoppel | a4c96ae | 2012-08-09 15:34:47 -0700 | [diff] [blame] | 4827 | static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 4828 | |
| 4829 | #endif /* CONFIG_CFS_BANDWIDTH */ |
| 4830 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4831 | /************************************************** |
| 4832 | * CFS operations on tasks: |
| 4833 | */ |
| 4834 | |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4835 | #ifdef CONFIG_SCHED_HRTICK |
| 4836 | static void hrtick_start_fair(struct rq *rq, struct task_struct *p) |
| 4837 | { |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4838 | struct sched_entity *se = &p->se; |
| 4839 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 4840 | |
Peter Zijlstra | 9148a3a | 2016-09-20 22:34:51 +0200 | [diff] [blame] | 4841 | SCHED_WARN_ON(task_rq(p) != rq); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4842 | |
Srivatsa Vaddagiri | 8bf46a3 | 2016-09-16 18:28:51 -0700 | [diff] [blame] | 4843 | if (rq->cfs.h_nr_running > 1) { |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4844 | u64 slice = sched_slice(cfs_rq, se); |
| 4845 | u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; |
| 4846 | s64 delta = slice - ran; |
| 4847 | |
| 4848 | if (delta < 0) { |
| 4849 | if (rq->curr == p) |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 4850 | resched_curr(rq); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4851 | return; |
| 4852 | } |
Peter Zijlstra | 3165651 | 2008-07-18 18:01:23 +0200 | [diff] [blame] | 4853 | hrtick_start(rq, delta); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4854 | } |
| 4855 | } |
Peter Zijlstra | a4c2f00 | 2008-10-17 19:27:03 +0200 | [diff] [blame] | 4856 | |
| 4857 | /* |
| 4858 | * called from enqueue/dequeue and updates the hrtick when the |
| 4859 | * current task is from our class and nr_running is low enough |
| 4860 | * to matter. |
| 4861 | */ |
| 4862 | static void hrtick_update(struct rq *rq) |
| 4863 | { |
| 4864 | struct task_struct *curr = rq->curr; |
| 4865 | |
Mike Galbraith | b39e66e | 2011-11-22 15:20:07 +0100 | [diff] [blame] | 4866 | if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) |
Peter Zijlstra | a4c2f00 | 2008-10-17 19:27:03 +0200 | [diff] [blame] | 4867 | return; |
| 4868 | |
| 4869 | if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) |
| 4870 | hrtick_start_fair(rq, curr); |
| 4871 | } |
Dhaval Giani | 55e12e5 | 2008-06-24 23:39:43 +0530 | [diff] [blame] | 4872 | #else /* !CONFIG_SCHED_HRTICK */ |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4873 | static inline void |
| 4874 | hrtick_start_fair(struct rq *rq, struct task_struct *p) |
| 4875 | { |
| 4876 | } |
Peter Zijlstra | a4c2f00 | 2008-10-17 19:27:03 +0200 | [diff] [blame] | 4877 | |
| 4878 | static inline void hrtick_update(struct rq *rq) |
| 4879 | { |
| 4880 | } |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4881 | #endif |
| 4882 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4883 | /* |
| 4884 | * The enqueue_task method is called before nr_running is |
| 4885 | * increased. Here we update the fair scheduling stats and |
| 4886 | * then put the task into the rbtree: |
| 4887 | */ |
Thomas Gleixner | ea87bb7 | 2010-01-20 20:58:57 +0000 | [diff] [blame] | 4888 | static void |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 4889 | enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4890 | { |
| 4891 | struct cfs_rq *cfs_rq; |
Peter Zijlstra | 62fb185 | 2008-02-25 17:34:02 +0100 | [diff] [blame] | 4892 | struct sched_entity *se = &p->se; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4893 | |
Rafael J. Wysocki | 8c34ab1 | 2016-09-09 23:59:33 +0200 | [diff] [blame] | 4894 | /* |
| 4895 | * If in_iowait is set, the code below may not trigger any cpufreq |
| 4896 | * utilization updates, so do it here explicitly with the IOWAIT flag |
| 4897 | * passed. |
| 4898 | */ |
| 4899 | if (p->in_iowait) |
| 4900 | cpufreq_update_this_cpu(rq, SCHED_CPUFREQ_IOWAIT); |
| 4901 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4902 | for_each_sched_entity(se) { |
Peter Zijlstra | 62fb185 | 2008-02-25 17:34:02 +0100 | [diff] [blame] | 4903 | if (se->on_rq) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4904 | break; |
| 4905 | cfs_rq = cfs_rq_of(se); |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 4906 | enqueue_entity(cfs_rq, se, flags); |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4907 | |
| 4908 | /* |
| 4909 | * end evaluation on encountering a throttled cfs_rq |
| 4910 | * |
| 4911 | * note: in the case of encountering a throttled cfs_rq we will |
| 4912 | * post the final h_nr_running increment below. |
Peter Zijlstra | e210bff | 2016-06-16 18:51:48 +0200 | [diff] [blame] | 4913 | */ |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4914 | if (cfs_rq_throttled(cfs_rq)) |
| 4915 | break; |
Paul Turner | 953bfcd | 2011-07-21 09:43:27 -0700 | [diff] [blame] | 4916 | cfs_rq->h_nr_running++; |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4917 | |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 4918 | flags = ENQUEUE_WAKEUP; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4919 | } |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4920 | |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 4921 | for_each_sched_entity(se) { |
Lin Ming | 0f31714 | 2011-07-22 09:14:31 +0800 | [diff] [blame] | 4922 | cfs_rq = cfs_rq_of(se); |
Paul Turner | 953bfcd | 2011-07-21 09:43:27 -0700 | [diff] [blame] | 4923 | cfs_rq->h_nr_running++; |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 4924 | |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4925 | if (cfs_rq_throttled(cfs_rq)) |
| 4926 | break; |
| 4927 | |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 4928 | update_load_avg(se, UPDATE_TG); |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 4929 | update_cfs_shares(se); |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 4930 | } |
| 4931 | |
Yuyang Du | cd126af | 2015-07-15 08:04:36 +0800 | [diff] [blame] | 4932 | if (!se) |
Kirill Tkhai | 7246544 | 2014-05-09 03:00:14 +0400 | [diff] [blame] | 4933 | add_nr_running(rq, 1); |
Yuyang Du | cd126af | 2015-07-15 08:04:36 +0800 | [diff] [blame] | 4934 | |
Peter Zijlstra | a4c2f00 | 2008-10-17 19:27:03 +0200 | [diff] [blame] | 4935 | hrtick_update(rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4936 | } |
| 4937 | |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 4938 | static void set_next_buddy(struct sched_entity *se); |
| 4939 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4940 | /* |
| 4941 | * The dequeue_task method is called before nr_running is |
| 4942 | * decreased. We remove the task from the rbtree and |
| 4943 | * update the fair scheduling stats: |
| 4944 | */ |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 4945 | static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4946 | { |
| 4947 | struct cfs_rq *cfs_rq; |
Peter Zijlstra | 62fb185 | 2008-02-25 17:34:02 +0100 | [diff] [blame] | 4948 | struct sched_entity *se = &p->se; |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 4949 | int task_sleep = flags & DEQUEUE_SLEEP; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4950 | |
| 4951 | for_each_sched_entity(se) { |
| 4952 | cfs_rq = cfs_rq_of(se); |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 4953 | dequeue_entity(cfs_rq, se, flags); |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4954 | |
| 4955 | /* |
| 4956 | * end evaluation on encountering a throttled cfs_rq |
| 4957 | * |
| 4958 | * note: in the case of encountering a throttled cfs_rq we will |
| 4959 | * post the final h_nr_running decrement below. |
| 4960 | */ |
| 4961 | if (cfs_rq_throttled(cfs_rq)) |
| 4962 | break; |
Paul Turner | 953bfcd | 2011-07-21 09:43:27 -0700 | [diff] [blame] | 4963 | cfs_rq->h_nr_running--; |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 4964 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4965 | /* Don't dequeue parent if it has other entities besides us */ |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 4966 | if (cfs_rq->load.weight) { |
Konstantin Khlebnikov | 754bd59 | 2016-06-16 15:57:15 +0300 | [diff] [blame] | 4967 | /* Avoid re-evaluating load for this entity: */ |
| 4968 | se = parent_entity(se); |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 4969 | /* |
| 4970 | * Bias pick_next to pick a task from this cfs_rq, as |
| 4971 | * p is sleeping when it is within its sched_slice. |
| 4972 | */ |
Konstantin Khlebnikov | 754bd59 | 2016-06-16 15:57:15 +0300 | [diff] [blame] | 4973 | if (task_sleep && se && !throttled_hierarchy(cfs_rq)) |
| 4974 | set_next_buddy(se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4975 | break; |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 4976 | } |
Peter Zijlstra | 371fd7e | 2010-03-24 16:38:48 +0100 | [diff] [blame] | 4977 | flags |= DEQUEUE_SLEEP; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4978 | } |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 4979 | |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 4980 | for_each_sched_entity(se) { |
Lin Ming | 0f31714 | 2011-07-22 09:14:31 +0800 | [diff] [blame] | 4981 | cfs_rq = cfs_rq_of(se); |
Paul Turner | 953bfcd | 2011-07-21 09:43:27 -0700 | [diff] [blame] | 4982 | cfs_rq->h_nr_running--; |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 4983 | |
Paul Turner | 85dac90 | 2011-07-21 09:43:33 -0700 | [diff] [blame] | 4984 | if (cfs_rq_throttled(cfs_rq)) |
| 4985 | break; |
| 4986 | |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 4987 | update_load_avg(se, UPDATE_TG); |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 4988 | update_cfs_shares(se); |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 4989 | } |
| 4990 | |
Yuyang Du | cd126af | 2015-07-15 08:04:36 +0800 | [diff] [blame] | 4991 | if (!se) |
Kirill Tkhai | 7246544 | 2014-05-09 03:00:14 +0400 | [diff] [blame] | 4992 | sub_nr_running(rq, 1); |
Yuyang Du | cd126af | 2015-07-15 08:04:36 +0800 | [diff] [blame] | 4993 | |
Peter Zijlstra | a4c2f00 | 2008-10-17 19:27:03 +0200 | [diff] [blame] | 4994 | hrtick_update(rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 4995 | } |
| 4996 | |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 4997 | #ifdef CONFIG_SMP |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 4998 | |
| 4999 | /* Working cpumask for: load_balance, load_balance_newidle. */ |
| 5000 | DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); |
| 5001 | DEFINE_PER_CPU(cpumask_var_t, select_idle_mask); |
| 5002 | |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 5003 | #ifdef CONFIG_NO_HZ_COMMON |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5004 | /* |
| 5005 | * per rq 'load' arrray crap; XXX kill this. |
| 5006 | */ |
| 5007 | |
| 5008 | /* |
Peter Zijlstra | d937cdc | 2015-10-19 13:49:30 +0200 | [diff] [blame] | 5009 | * The exact cpuload calculated at every tick would be: |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5010 | * |
Peter Zijlstra | d937cdc | 2015-10-19 13:49:30 +0200 | [diff] [blame] | 5011 | * load' = (1 - 1/2^i) * load + (1/2^i) * cur_load |
| 5012 | * |
| 5013 | * If a cpu misses updates for n ticks (as it was idle) and update gets |
| 5014 | * called on the n+1-th tick when cpu may be busy, then we have: |
| 5015 | * |
| 5016 | * load_n = (1 - 1/2^i)^n * load_0 |
| 5017 | * load_n+1 = (1 - 1/2^i) * load_n + (1/2^i) * cur_load |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5018 | * |
| 5019 | * decay_load_missed() below does efficient calculation of |
Peter Zijlstra | d937cdc | 2015-10-19 13:49:30 +0200 | [diff] [blame] | 5020 | * |
| 5021 | * load' = (1 - 1/2^i)^n * load |
| 5022 | * |
| 5023 | * Because x^(n+m) := x^n * x^m we can decompose any x^n in power-of-2 factors. |
| 5024 | * This allows us to precompute the above in said factors, thereby allowing the |
| 5025 | * reduction of an arbitrary n in O(log_2 n) steps. (See also |
| 5026 | * fixed_power_int()) |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5027 | * |
| 5028 | * The calculation is approximated on a 128 point scale. |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5029 | */ |
| 5030 | #define DEGRADE_SHIFT 7 |
Peter Zijlstra | d937cdc | 2015-10-19 13:49:30 +0200 | [diff] [blame] | 5031 | |
| 5032 | static const u8 degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; |
| 5033 | static const u8 degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { |
| 5034 | { 0, 0, 0, 0, 0, 0, 0, 0 }, |
| 5035 | { 64, 32, 8, 0, 0, 0, 0, 0 }, |
| 5036 | { 96, 72, 40, 12, 1, 0, 0, 0 }, |
| 5037 | { 112, 98, 75, 43, 15, 1, 0, 0 }, |
| 5038 | { 120, 112, 98, 76, 45, 16, 2, 0 } |
| 5039 | }; |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5040 | |
| 5041 | /* |
| 5042 | * Update cpu_load for any missed ticks, due to tickless idle. The backlog |
| 5043 | * would be when CPU is idle and so we just decay the old load without |
| 5044 | * adding any new load. |
| 5045 | */ |
| 5046 | static unsigned long |
| 5047 | decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) |
| 5048 | { |
| 5049 | int j = 0; |
| 5050 | |
| 5051 | if (!missed_updates) |
| 5052 | return load; |
| 5053 | |
| 5054 | if (missed_updates >= degrade_zero_ticks[idx]) |
| 5055 | return 0; |
| 5056 | |
| 5057 | if (idx == 1) |
| 5058 | return load >> missed_updates; |
| 5059 | |
| 5060 | while (missed_updates) { |
| 5061 | if (missed_updates % 2) |
| 5062 | load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; |
| 5063 | |
| 5064 | missed_updates >>= 1; |
| 5065 | j++; |
| 5066 | } |
| 5067 | return load; |
| 5068 | } |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 5069 | #endif /* CONFIG_NO_HZ_COMMON */ |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5070 | |
Byungchul Park | 5954327 | 2015-10-14 18:47:35 +0900 | [diff] [blame] | 5071 | /** |
Frederic Weisbecker | cee1afc | 2016-04-13 15:56:50 +0200 | [diff] [blame] | 5072 | * __cpu_load_update - update the rq->cpu_load[] statistics |
Byungchul Park | 5954327 | 2015-10-14 18:47:35 +0900 | [diff] [blame] | 5073 | * @this_rq: The rq to update statistics for |
| 5074 | * @this_load: The current load |
| 5075 | * @pending_updates: The number of missed updates |
Byungchul Park | 5954327 | 2015-10-14 18:47:35 +0900 | [diff] [blame] | 5076 | * |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5077 | * Update rq->cpu_load[] statistics. This function is usually called every |
Byungchul Park | 5954327 | 2015-10-14 18:47:35 +0900 | [diff] [blame] | 5078 | * scheduler tick (TICK_NSEC). |
| 5079 | * |
| 5080 | * This function computes a decaying average: |
| 5081 | * |
| 5082 | * load[i]' = (1 - 1/2^i) * load[i] + (1/2^i) * load |
| 5083 | * |
| 5084 | * Because of NOHZ it might not get called on every tick which gives need for |
| 5085 | * the @pending_updates argument. |
| 5086 | * |
| 5087 | * load[i]_n = (1 - 1/2^i) * load[i]_n-1 + (1/2^i) * load_n-1 |
| 5088 | * = A * load[i]_n-1 + B ; A := (1 - 1/2^i), B := (1/2^i) * load |
| 5089 | * = A * (A * load[i]_n-2 + B) + B |
| 5090 | * = A * (A * (A * load[i]_n-3 + B) + B) + B |
| 5091 | * = A^3 * load[i]_n-3 + (A^2 + A + 1) * B |
| 5092 | * = A^n * load[i]_0 + (A^(n-1) + A^(n-2) + ... + 1) * B |
| 5093 | * = A^n * load[i]_0 + ((1 - A^n) / (1 - A)) * B |
| 5094 | * = (1 - 1/2^i)^n * (load[i]_0 - load) + load |
| 5095 | * |
| 5096 | * In the above we've assumed load_n := load, which is true for NOHZ_FULL as |
| 5097 | * any change in load would have resulted in the tick being turned back on. |
| 5098 | * |
| 5099 | * For regular NOHZ, this reduces to: |
| 5100 | * |
| 5101 | * load[i]_n = (1 - 1/2^i)^n * load[i]_0 |
| 5102 | * |
| 5103 | * see decay_load_misses(). For NOHZ_FULL we get to subtract and add the extra |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5104 | * term. |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5105 | */ |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5106 | static void cpu_load_update(struct rq *this_rq, unsigned long this_load, |
| 5107 | unsigned long pending_updates) |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5108 | { |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 5109 | unsigned long __maybe_unused tickless_load = this_rq->cpu_load[0]; |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5110 | int i, scale; |
| 5111 | |
| 5112 | this_rq->nr_load_updates++; |
| 5113 | |
| 5114 | /* Update our load: */ |
| 5115 | this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ |
| 5116 | for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { |
| 5117 | unsigned long old_load, new_load; |
| 5118 | |
| 5119 | /* scale is effectively 1 << i now, and >> i divides by scale */ |
| 5120 | |
Byungchul Park | 7400d3b | 2016-01-15 16:07:49 +0900 | [diff] [blame] | 5121 | old_load = this_rq->cpu_load[i]; |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 5122 | #ifdef CONFIG_NO_HZ_COMMON |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5123 | old_load = decay_load_missed(old_load, pending_updates - 1, i); |
Byungchul Park | 7400d3b | 2016-01-15 16:07:49 +0900 | [diff] [blame] | 5124 | if (tickless_load) { |
| 5125 | old_load -= decay_load_missed(tickless_load, pending_updates - 1, i); |
| 5126 | /* |
| 5127 | * old_load can never be a negative value because a |
| 5128 | * decayed tickless_load cannot be greater than the |
| 5129 | * original tickless_load. |
| 5130 | */ |
| 5131 | old_load += tickless_load; |
| 5132 | } |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 5133 | #endif |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5134 | new_load = this_load; |
| 5135 | /* |
| 5136 | * Round up the averaging division if load is increasing. This |
| 5137 | * prevents us from getting stuck on 9 if the load is 10, for |
| 5138 | * example. |
| 5139 | */ |
| 5140 | if (new_load > old_load) |
| 5141 | new_load += scale - 1; |
| 5142 | |
| 5143 | this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; |
| 5144 | } |
| 5145 | |
| 5146 | sched_avg_update(this_rq); |
| 5147 | } |
| 5148 | |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 5149 | /* Used instead of source_load when we know the type == 0 */ |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5150 | static unsigned long weighted_cpuload(struct rq *rq) |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 5151 | { |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5152 | return cfs_rq_runnable_load_avg(&rq->cfs); |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 5153 | } |
| 5154 | |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5155 | #ifdef CONFIG_NO_HZ_COMMON |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5156 | /* |
| 5157 | * There is no sane way to deal with nohz on smp when using jiffies because the |
| 5158 | * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading |
| 5159 | * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. |
| 5160 | * |
| 5161 | * Therefore we need to avoid the delta approach from the regular tick when |
| 5162 | * possible since that would seriously skew the load calculation. This is why we |
| 5163 | * use cpu_load_update_periodic() for CPUs out of nohz. However we'll rely on |
| 5164 | * jiffies deltas for updates happening while in nohz mode (idle ticks, idle |
| 5165 | * loop exit, nohz_idle_balance, nohz full exit...) |
| 5166 | * |
| 5167 | * This means we might still be one tick off for nohz periods. |
| 5168 | */ |
| 5169 | |
| 5170 | static void cpu_load_update_nohz(struct rq *this_rq, |
| 5171 | unsigned long curr_jiffies, |
| 5172 | unsigned long load) |
Frederic Weisbecker | be68a68 | 2016-01-13 17:01:29 +0100 | [diff] [blame] | 5173 | { |
| 5174 | unsigned long pending_updates; |
| 5175 | |
| 5176 | pending_updates = curr_jiffies - this_rq->last_load_update_tick; |
| 5177 | if (pending_updates) { |
| 5178 | this_rq->last_load_update_tick = curr_jiffies; |
| 5179 | /* |
| 5180 | * In the regular NOHZ case, we were idle, this means load 0. |
| 5181 | * In the NOHZ_FULL case, we were non-idle, we should consider |
| 5182 | * its weighted load. |
| 5183 | */ |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5184 | cpu_load_update(this_rq, load, pending_updates); |
Frederic Weisbecker | be68a68 | 2016-01-13 17:01:29 +0100 | [diff] [blame] | 5185 | } |
| 5186 | } |
| 5187 | |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5188 | /* |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5189 | * Called from nohz_idle_balance() to update the load ratings before doing the |
| 5190 | * idle balance. |
| 5191 | */ |
Frederic Weisbecker | cee1afc | 2016-04-13 15:56:50 +0200 | [diff] [blame] | 5192 | static void cpu_load_update_idle(struct rq *this_rq) |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5193 | { |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5194 | /* |
| 5195 | * bail if there's load or we're actually up-to-date. |
| 5196 | */ |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5197 | if (weighted_cpuload(this_rq)) |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5198 | return; |
| 5199 | |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5200 | cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), 0); |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5201 | } |
| 5202 | |
| 5203 | /* |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5204 | * Record CPU load on nohz entry so we know the tickless load to account |
| 5205 | * on nohz exit. cpu_load[0] happens then to be updated more frequently |
| 5206 | * than other cpu_load[idx] but it should be fine as cpu_load readers |
| 5207 | * shouldn't rely into synchronized cpu_load[*] updates. |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5208 | */ |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5209 | void cpu_load_update_nohz_start(void) |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5210 | { |
| 5211 | struct rq *this_rq = this_rq(); |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5212 | |
| 5213 | /* |
| 5214 | * This is all lockless but should be fine. If weighted_cpuload changes |
| 5215 | * concurrently we'll exit nohz. And cpu_load write can race with |
| 5216 | * cpu_load_update_idle() but both updater would be writing the same. |
| 5217 | */ |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5218 | this_rq->cpu_load[0] = weighted_cpuload(this_rq); |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5219 | } |
| 5220 | |
| 5221 | /* |
| 5222 | * Account the tickless load in the end of a nohz frame. |
| 5223 | */ |
| 5224 | void cpu_load_update_nohz_stop(void) |
| 5225 | { |
Jason Low | 316c1608d | 2015-04-28 13:00:20 -0700 | [diff] [blame] | 5226 | unsigned long curr_jiffies = READ_ONCE(jiffies); |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5227 | struct rq *this_rq = this_rq(); |
| 5228 | unsigned long load; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 5229 | struct rq_flags rf; |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5230 | |
| 5231 | if (curr_jiffies == this_rq->last_load_update_tick) |
| 5232 | return; |
| 5233 | |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5234 | load = weighted_cpuload(this_rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 5235 | rq_lock(this_rq, &rf); |
Matt Fleming | b52fad2 | 2016-05-03 20:46:54 +0100 | [diff] [blame] | 5236 | update_rq_clock(this_rq); |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5237 | cpu_load_update_nohz(this_rq, curr_jiffies, load); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 5238 | rq_unlock(this_rq, &rf); |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5239 | } |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5240 | #else /* !CONFIG_NO_HZ_COMMON */ |
| 5241 | static inline void cpu_load_update_nohz(struct rq *this_rq, |
| 5242 | unsigned long curr_jiffies, |
| 5243 | unsigned long load) { } |
| 5244 | #endif /* CONFIG_NO_HZ_COMMON */ |
| 5245 | |
| 5246 | static void cpu_load_update_periodic(struct rq *this_rq, unsigned long load) |
| 5247 | { |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 5248 | #ifdef CONFIG_NO_HZ_COMMON |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5249 | /* See the mess around cpu_load_update_nohz(). */ |
| 5250 | this_rq->last_load_update_tick = READ_ONCE(jiffies); |
Frederic Weisbecker | 9fd81dd | 2016-04-19 17:36:51 +0200 | [diff] [blame] | 5251 | #endif |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5252 | cpu_load_update(this_rq, load, 1); |
| 5253 | } |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5254 | |
| 5255 | /* |
| 5256 | * Called from scheduler_tick() |
| 5257 | */ |
Frederic Weisbecker | cee1afc | 2016-04-13 15:56:50 +0200 | [diff] [blame] | 5258 | void cpu_load_update_active(struct rq *this_rq) |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5259 | { |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5260 | unsigned long load = weighted_cpuload(this_rq); |
Frederic Weisbecker | 1f41906 | 2016-04-13 15:56:51 +0200 | [diff] [blame] | 5261 | |
| 5262 | if (tick_nohz_tick_stopped()) |
| 5263 | cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), load); |
| 5264 | else |
| 5265 | cpu_load_update_periodic(this_rq, load); |
Peter Zijlstra | 3289bdb | 2015-04-14 13:19:42 +0200 | [diff] [blame] | 5266 | } |
| 5267 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5268 | /* |
| 5269 | * Return a low guess at the load of a migration-source cpu weighted |
| 5270 | * according to the scheduling class and "nice" value. |
| 5271 | * |
| 5272 | * We want to under-estimate the load of migration sources, to |
| 5273 | * balance conservatively. |
| 5274 | */ |
| 5275 | static unsigned long source_load(int cpu, int type) |
| 5276 | { |
| 5277 | struct rq *rq = cpu_rq(cpu); |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5278 | unsigned long total = weighted_cpuload(rq); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5279 | |
| 5280 | if (type == 0 || !sched_feat(LB_BIAS)) |
| 5281 | return total; |
| 5282 | |
| 5283 | return min(rq->cpu_load[type-1], total); |
| 5284 | } |
| 5285 | |
| 5286 | /* |
| 5287 | * Return a high guess at the load of a migration-target cpu weighted |
| 5288 | * according to the scheduling class and "nice" value. |
| 5289 | */ |
| 5290 | static unsigned long target_load(int cpu, int type) |
| 5291 | { |
| 5292 | struct rq *rq = cpu_rq(cpu); |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5293 | unsigned long total = weighted_cpuload(rq); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5294 | |
| 5295 | if (type == 0 || !sched_feat(LB_BIAS)) |
| 5296 | return total; |
| 5297 | |
| 5298 | return max(rq->cpu_load[type-1], total); |
| 5299 | } |
| 5300 | |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 5301 | static unsigned long capacity_of(int cpu) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5302 | { |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 5303 | return cpu_rq(cpu)->cpu_capacity; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5304 | } |
| 5305 | |
Vincent Guittot | ca6d75e | 2015-02-27 16:54:09 +0100 | [diff] [blame] | 5306 | static unsigned long capacity_orig_of(int cpu) |
| 5307 | { |
| 5308 | return cpu_rq(cpu)->cpu_capacity_orig; |
| 5309 | } |
| 5310 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5311 | static unsigned long cpu_avg_load_per_task(int cpu) |
| 5312 | { |
| 5313 | struct rq *rq = cpu_rq(cpu); |
Jason Low | 316c1608d | 2015-04-28 13:00:20 -0700 | [diff] [blame] | 5314 | unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running); |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5315 | unsigned long load_avg = weighted_cpuload(rq); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5316 | |
| 5317 | if (nr_running) |
Alex Shi | b92486c | 2013-06-20 10:18:50 +0800 | [diff] [blame] | 5318 | return load_avg / nr_running; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 5319 | |
| 5320 | return 0; |
| 5321 | } |
| 5322 | |
Peter Zijlstra | c58d25f | 2016-05-12 09:19:59 +0200 | [diff] [blame] | 5323 | static void record_wakee(struct task_struct *p) |
| 5324 | { |
| 5325 | /* |
| 5326 | * Only decay a single time; tasks that have less then 1 wakeup per |
| 5327 | * jiffy will not have built up many flips. |
| 5328 | */ |
| 5329 | if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { |
| 5330 | current->wakee_flips >>= 1; |
| 5331 | current->wakee_flip_decay_ts = jiffies; |
| 5332 | } |
| 5333 | |
| 5334 | if (current->last_wakee != p) { |
| 5335 | current->last_wakee = p; |
| 5336 | current->wakee_flips++; |
| 5337 | } |
| 5338 | } |
| 5339 | |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 5340 | /* |
| 5341 | * Detect M:N waker/wakee relationships via a switching-frequency heuristic. |
Peter Zijlstra | c58d25f | 2016-05-12 09:19:59 +0200 | [diff] [blame] | 5342 | * |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 5343 | * A waker of many should wake a different task than the one last awakened |
Peter Zijlstra | c58d25f | 2016-05-12 09:19:59 +0200 | [diff] [blame] | 5344 | * at a frequency roughly N times higher than one of its wakees. |
| 5345 | * |
| 5346 | * In order to determine whether we should let the load spread vs consolidating |
| 5347 | * to shared cache, we look for a minimum 'flip' frequency of llc_size in one |
| 5348 | * partner, and a factor of lls_size higher frequency in the other. |
| 5349 | * |
| 5350 | * With both conditions met, we can be relatively sure that the relationship is |
| 5351 | * non-monogamous, with partner count exceeding socket size. |
| 5352 | * |
| 5353 | * Waker/wakee being client/server, worker/dispatcher, interrupt source or |
| 5354 | * whatever is irrelevant, spread criteria is apparent partner count exceeds |
| 5355 | * socket size. |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 5356 | */ |
Michael Wang | 6247041 | 2013-07-04 12:55:51 +0800 | [diff] [blame] | 5357 | static int wake_wide(struct task_struct *p) |
| 5358 | { |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 5359 | unsigned int master = current->wakee_flips; |
| 5360 | unsigned int slave = p->wakee_flips; |
Peter Zijlstra | 7d9ffa8 | 2013-07-04 12:56:46 +0800 | [diff] [blame] | 5361 | int factor = this_cpu_read(sd_llc_size); |
Michael Wang | 6247041 | 2013-07-04 12:55:51 +0800 | [diff] [blame] | 5362 | |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 5363 | if (master < slave) |
| 5364 | swap(master, slave); |
| 5365 | if (slave < factor || master < slave * factor) |
| 5366 | return 0; |
| 5367 | return 1; |
Michael Wang | 6247041 | 2013-07-04 12:55:51 +0800 | [diff] [blame] | 5368 | } |
| 5369 | |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 5370 | struct llc_stats { |
| 5371 | unsigned long nr_running; |
| 5372 | unsigned long load; |
| 5373 | unsigned long capacity; |
| 5374 | int has_capacity; |
| 5375 | }; |
| 5376 | |
| 5377 | static bool get_llc_stats(struct llc_stats *stats, int cpu) |
| 5378 | { |
| 5379 | struct sched_domain_shared *sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); |
| 5380 | |
| 5381 | if (!sds) |
| 5382 | return false; |
| 5383 | |
| 5384 | stats->nr_running = READ_ONCE(sds->nr_running); |
| 5385 | stats->load = READ_ONCE(sds->load); |
| 5386 | stats->capacity = READ_ONCE(sds->capacity); |
| 5387 | stats->has_capacity = stats->nr_running < per_cpu(sd_llc_size, cpu); |
| 5388 | |
| 5389 | return true; |
| 5390 | } |
| 5391 | |
| 5392 | /* |
| 5393 | * Can a task be moved from prev_cpu to this_cpu without causing a load |
| 5394 | * imbalance that would trigger the load balancer? |
| 5395 | * |
| 5396 | * Since we're running on 'stale' values, we might in fact create an imbalance |
| 5397 | * but recomputing these values is expensive, as that'd mean iteration 2 cache |
| 5398 | * domains worth of CPUs. |
| 5399 | */ |
| 5400 | static bool |
| 5401 | wake_affine_llc(struct sched_domain *sd, struct task_struct *p, |
| 5402 | int this_cpu, int prev_cpu, int sync) |
| 5403 | { |
| 5404 | struct llc_stats prev_stats, this_stats; |
| 5405 | s64 this_eff_load, prev_eff_load; |
| 5406 | unsigned long task_load; |
| 5407 | |
| 5408 | if (!get_llc_stats(&prev_stats, prev_cpu) || |
| 5409 | !get_llc_stats(&this_stats, this_cpu)) |
| 5410 | return false; |
| 5411 | |
| 5412 | /* |
| 5413 | * If sync wakeup then subtract the (maximum possible) |
| 5414 | * effect of the currently running task from the load |
| 5415 | * of the current LLC. |
| 5416 | */ |
| 5417 | if (sync) { |
| 5418 | unsigned long current_load = task_h_load(current); |
| 5419 | |
| 5420 | /* in this case load hits 0 and this LLC is considered 'idle' */ |
| 5421 | if (current_load > this_stats.load) |
| 5422 | return true; |
| 5423 | |
| 5424 | this_stats.load -= current_load; |
| 5425 | } |
| 5426 | |
| 5427 | /* |
| 5428 | * The has_capacity stuff is not SMT aware, but by trying to balance |
| 5429 | * the nr_running on both ends we try and fill the domain at equal |
| 5430 | * rates, thereby first consuming cores before siblings. |
| 5431 | */ |
| 5432 | |
| 5433 | /* if the old cache has capacity, stay there */ |
| 5434 | if (prev_stats.has_capacity && prev_stats.nr_running < this_stats.nr_running+1) |
| 5435 | return false; |
| 5436 | |
| 5437 | /* if this cache has capacity, come here */ |
| 5438 | if (this_stats.has_capacity && this_stats.nr_running < prev_stats.nr_running+1) |
| 5439 | return true; |
| 5440 | |
| 5441 | /* |
| 5442 | * Check to see if we can move the load without causing too much |
| 5443 | * imbalance. |
| 5444 | */ |
| 5445 | task_load = task_h_load(p); |
| 5446 | |
| 5447 | this_eff_load = 100; |
| 5448 | this_eff_load *= prev_stats.capacity; |
| 5449 | |
| 5450 | prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; |
| 5451 | prev_eff_load *= this_stats.capacity; |
| 5452 | |
| 5453 | this_eff_load *= this_stats.load + task_load; |
| 5454 | prev_eff_load *= prev_stats.load - task_load; |
| 5455 | |
| 5456 | return this_eff_load <= prev_eff_load; |
| 5457 | } |
| 5458 | |
Morten Rasmussen | 772bd008c | 2016-06-22 18:03:13 +0100 | [diff] [blame] | 5459 | static int wake_affine(struct sched_domain *sd, struct task_struct *p, |
| 5460 | int prev_cpu, int sync) |
Ingo Molnar | 098fb9d | 2008-03-16 20:36:10 +0100 | [diff] [blame] | 5461 | { |
Rik van Riel | 3fed382 | 2017-06-23 12:55:29 -0400 | [diff] [blame] | 5462 | int this_cpu = smp_processor_id(); |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 5463 | bool affine; |
Ingo Molnar | 098fb9d | 2008-03-16 20:36:10 +0100 | [diff] [blame] | 5464 | |
| 5465 | /* |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 5466 | * Default to no affine wakeups; wake_affine() should not effect a task |
| 5467 | * placement the load-balancer feels inclined to undo. The conservative |
| 5468 | * option is therefore to not move tasks when they wake up. |
Rik van Riel | 7d894e6 | 2017-06-23 12:55:28 -0400 | [diff] [blame] | 5469 | */ |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 5470 | affine = false; |
| 5471 | |
| 5472 | /* |
| 5473 | * If the wakeup is across cache domains, try to evaluate if movement |
| 5474 | * makes sense, otherwise rely on select_idle_siblings() to do |
| 5475 | * placement inside the cache domain. |
| 5476 | */ |
| 5477 | if (!cpus_share_cache(prev_cpu, this_cpu)) |
| 5478 | affine = wake_affine_llc(sd, p, this_cpu, prev_cpu, sync); |
Mike Galbraith | b3137bc | 2008-05-29 11:11:41 +0200 | [diff] [blame] | 5479 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 5480 | schedstat_inc(p->se.statistics.nr_wakeups_affine_attempts); |
Rik van Riel | 3fed382 | 2017-06-23 12:55:29 -0400 | [diff] [blame] | 5481 | if (affine) { |
| 5482 | schedstat_inc(sd->ttwu_move_affine); |
| 5483 | schedstat_inc(p->se.statistics.nr_wakeups_affine); |
| 5484 | } |
Mike Galbraith | b3137bc | 2008-05-29 11:11:41 +0200 | [diff] [blame] | 5485 | |
Rik van Riel | 3fed382 | 2017-06-23 12:55:29 -0400 | [diff] [blame] | 5486 | return affine; |
Ingo Molnar | 098fb9d | 2008-03-16 20:36:10 +0100 | [diff] [blame] | 5487 | } |
| 5488 | |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5489 | static inline int task_util(struct task_struct *p); |
| 5490 | static int cpu_util_wake(int cpu, struct task_struct *p); |
| 5491 | |
| 5492 | static unsigned long capacity_spare_wake(int cpu, struct task_struct *p) |
| 5493 | { |
| 5494 | return capacity_orig_of(cpu) - cpu_util_wake(cpu, p); |
| 5495 | } |
| 5496 | |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5497 | /* |
| 5498 | * find_idlest_group finds and returns the least busy CPU group within the |
| 5499 | * domain. |
| 5500 | */ |
| 5501 | static struct sched_group * |
Peter Zijlstra | 78e7ed5 | 2009-09-03 13:16:51 +0200 | [diff] [blame] | 5502 | find_idlest_group(struct sched_domain *sd, struct task_struct *p, |
Vincent Guittot | c44f2a0 | 2013-10-18 13:52:21 +0200 | [diff] [blame] | 5503 | int this_cpu, int sd_flag) |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 5504 | { |
Andi Kleen | b3bd3de | 2010-08-10 14:17:51 -0700 | [diff] [blame] | 5505 | struct sched_group *idlest = NULL, *group = sd->groups; |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5506 | struct sched_group *most_spare_sg = NULL; |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5507 | unsigned long min_runnable_load = ULONG_MAX, this_runnable_load = 0; |
| 5508 | unsigned long min_avg_load = ULONG_MAX, this_avg_load = 0; |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5509 | unsigned long most_spare = 0, this_spare = 0; |
Vincent Guittot | c44f2a0 | 2013-10-18 13:52:21 +0200 | [diff] [blame] | 5510 | int load_idx = sd->forkexec_idx; |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5511 | int imbalance_scale = 100 + (sd->imbalance_pct-100)/2; |
| 5512 | unsigned long imbalance = scale_load_down(NICE_0_LOAD) * |
| 5513 | (sd->imbalance_pct-100) / 100; |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 5514 | |
Vincent Guittot | c44f2a0 | 2013-10-18 13:52:21 +0200 | [diff] [blame] | 5515 | if (sd_flag & SD_BALANCE_WAKE) |
| 5516 | load_idx = sd->wake_idx; |
| 5517 | |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5518 | do { |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5519 | unsigned long load, avg_load, runnable_load; |
| 5520 | unsigned long spare_cap, max_spare_cap; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5521 | int local_group; |
| 5522 | int i; |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 5523 | |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5524 | /* Skip over this group if it has no CPUs allowed */ |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 5525 | if (!cpumask_intersects(sched_group_span(group), |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 5526 | &p->cpus_allowed)) |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5527 | continue; |
| 5528 | |
| 5529 | local_group = cpumask_test_cpu(this_cpu, |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 5530 | sched_group_span(group)); |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5531 | |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5532 | /* |
| 5533 | * Tally up the load of all CPUs in the group and find |
| 5534 | * the group containing the CPU with most spare capacity. |
| 5535 | */ |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5536 | avg_load = 0; |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5537 | runnable_load = 0; |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5538 | max_spare_cap = 0; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5539 | |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 5540 | for_each_cpu(i, sched_group_span(group)) { |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5541 | /* Bias balancing toward cpus of our domain */ |
| 5542 | if (local_group) |
| 5543 | load = source_load(i, load_idx); |
| 5544 | else |
| 5545 | load = target_load(i, load_idx); |
| 5546 | |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5547 | runnable_load += load; |
| 5548 | |
| 5549 | avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs); |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5550 | |
| 5551 | spare_cap = capacity_spare_wake(i, p); |
| 5552 | |
| 5553 | if (spare_cap > max_spare_cap) |
| 5554 | max_spare_cap = spare_cap; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5555 | } |
| 5556 | |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 5557 | /* Adjust by relative CPU capacity of the group */ |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5558 | avg_load = (avg_load * SCHED_CAPACITY_SCALE) / |
| 5559 | group->sgc->capacity; |
| 5560 | runnable_load = (runnable_load * SCHED_CAPACITY_SCALE) / |
| 5561 | group->sgc->capacity; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5562 | |
| 5563 | if (local_group) { |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5564 | this_runnable_load = runnable_load; |
| 5565 | this_avg_load = avg_load; |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5566 | this_spare = max_spare_cap; |
| 5567 | } else { |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5568 | if (min_runnable_load > (runnable_load + imbalance)) { |
| 5569 | /* |
| 5570 | * The runnable load is significantly smaller |
| 5571 | * so we can pick this new cpu |
| 5572 | */ |
| 5573 | min_runnable_load = runnable_load; |
| 5574 | min_avg_load = avg_load; |
| 5575 | idlest = group; |
| 5576 | } else if ((runnable_load < (min_runnable_load + imbalance)) && |
| 5577 | (100*min_avg_load > imbalance_scale*avg_load)) { |
| 5578 | /* |
| 5579 | * The runnable loads are close so take the |
| 5580 | * blocked load into account through avg_load. |
| 5581 | */ |
| 5582 | min_avg_load = avg_load; |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5583 | idlest = group; |
| 5584 | } |
| 5585 | |
| 5586 | if (most_spare < max_spare_cap) { |
| 5587 | most_spare = max_spare_cap; |
| 5588 | most_spare_sg = group; |
| 5589 | } |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5590 | } |
| 5591 | } while (group = group->next, group != sd->groups); |
| 5592 | |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5593 | /* |
| 5594 | * The cross-over point between using spare capacity or least load |
| 5595 | * is too conservative for high utilization tasks on partially |
| 5596 | * utilized systems if we require spare_capacity > task_util(p), |
| 5597 | * so we allow for some task stuffing by using |
| 5598 | * spare_capacity > task_util(p)/2. |
Vincent Guittot | f519a3f | 2016-12-08 17:56:53 +0100 | [diff] [blame] | 5599 | * |
| 5600 | * Spare capacity can't be used for fork because the utilization has |
| 5601 | * not been set yet, we must first select a rq to compute the initial |
| 5602 | * utilization. |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5603 | */ |
Vincent Guittot | f519a3f | 2016-12-08 17:56:53 +0100 | [diff] [blame] | 5604 | if (sd_flag & SD_BALANCE_FORK) |
| 5605 | goto skip_spare; |
| 5606 | |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5607 | if (this_spare > task_util(p) / 2 && |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5608 | imbalance_scale*this_spare > 100*most_spare) |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5609 | return NULL; |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5610 | |
| 5611 | if (most_spare > task_util(p) / 2) |
Morten Rasmussen | 6a0b19c | 2016-10-14 14:41:08 +0100 | [diff] [blame] | 5612 | return most_spare_sg; |
| 5613 | |
Vincent Guittot | f519a3f | 2016-12-08 17:56:53 +0100 | [diff] [blame] | 5614 | skip_spare: |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5615 | if (!idlest) |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5616 | return NULL; |
Vincent Guittot | 6b94780 | 2016-12-08 17:56:54 +0100 | [diff] [blame] | 5617 | |
| 5618 | if (min_runnable_load > (this_runnable_load + imbalance)) |
| 5619 | return NULL; |
| 5620 | |
| 5621 | if ((this_runnable_load < (min_runnable_load + imbalance)) && |
| 5622 | (100*this_avg_load < imbalance_scale*min_avg_load)) |
| 5623 | return NULL; |
| 5624 | |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5625 | return idlest; |
| 5626 | } |
| 5627 | |
| 5628 | /* |
| 5629 | * find_idlest_cpu - find the idlest cpu among the cpus in group. |
| 5630 | */ |
| 5631 | static int |
| 5632 | find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) |
| 5633 | { |
| 5634 | unsigned long load, min_load = ULONG_MAX; |
Nicolas Pitre | 83a0a96 | 2014-09-04 11:32:10 -0400 | [diff] [blame] | 5635 | unsigned int min_exit_latency = UINT_MAX; |
| 5636 | u64 latest_idle_timestamp = 0; |
| 5637 | int least_loaded_cpu = this_cpu; |
| 5638 | int shallowest_idle_cpu = -1; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5639 | int i; |
| 5640 | |
Morten Rasmussen | eaecf41 | 2016-06-22 18:03:14 +0100 | [diff] [blame] | 5641 | /* Check if we have any choice: */ |
| 5642 | if (group->group_weight == 1) |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 5643 | return cpumask_first(sched_group_span(group)); |
Morten Rasmussen | eaecf41 | 2016-06-22 18:03:14 +0100 | [diff] [blame] | 5644 | |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5645 | /* Traverse only the allowed CPUs */ |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 5646 | for_each_cpu_and(i, sched_group_span(group), &p->cpus_allowed) { |
Nicolas Pitre | 83a0a96 | 2014-09-04 11:32:10 -0400 | [diff] [blame] | 5647 | if (idle_cpu(i)) { |
| 5648 | struct rq *rq = cpu_rq(i); |
| 5649 | struct cpuidle_state *idle = idle_get_state(rq); |
| 5650 | if (idle && idle->exit_latency < min_exit_latency) { |
| 5651 | /* |
| 5652 | * We give priority to a CPU whose idle state |
| 5653 | * has the smallest exit latency irrespective |
| 5654 | * of any idle timestamp. |
| 5655 | */ |
| 5656 | min_exit_latency = idle->exit_latency; |
| 5657 | latest_idle_timestamp = rq->idle_stamp; |
| 5658 | shallowest_idle_cpu = i; |
| 5659 | } else if ((!idle || idle->exit_latency == min_exit_latency) && |
| 5660 | rq->idle_stamp > latest_idle_timestamp) { |
| 5661 | /* |
| 5662 | * If equal or no active idle state, then |
| 5663 | * the most recently idled CPU might have |
| 5664 | * a warmer cache. |
| 5665 | */ |
| 5666 | latest_idle_timestamp = rq->idle_stamp; |
| 5667 | shallowest_idle_cpu = i; |
| 5668 | } |
Yao Dongdong | 9f96742 | 2014-10-28 04:08:06 +0000 | [diff] [blame] | 5669 | } else if (shallowest_idle_cpu == -1) { |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 5670 | load = weighted_cpuload(cpu_rq(i)); |
Nicolas Pitre | 83a0a96 | 2014-09-04 11:32:10 -0400 | [diff] [blame] | 5671 | if (load < min_load || (load == min_load && i == this_cpu)) { |
| 5672 | min_load = load; |
| 5673 | least_loaded_cpu = i; |
| 5674 | } |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 5675 | } |
| 5676 | } |
| 5677 | |
Nicolas Pitre | 83a0a96 | 2014-09-04 11:32:10 -0400 | [diff] [blame] | 5678 | return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5679 | } |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 5680 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5681 | #ifdef CONFIG_SCHED_SMT |
| 5682 | |
| 5683 | static inline void set_idle_cores(int cpu, int val) |
| 5684 | { |
| 5685 | struct sched_domain_shared *sds; |
| 5686 | |
| 5687 | sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); |
| 5688 | if (sds) |
| 5689 | WRITE_ONCE(sds->has_idle_cores, val); |
| 5690 | } |
| 5691 | |
| 5692 | static inline bool test_idle_cores(int cpu, bool def) |
| 5693 | { |
| 5694 | struct sched_domain_shared *sds; |
| 5695 | |
| 5696 | sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); |
| 5697 | if (sds) |
| 5698 | return READ_ONCE(sds->has_idle_cores); |
| 5699 | |
| 5700 | return def; |
| 5701 | } |
| 5702 | |
| 5703 | /* |
| 5704 | * Scans the local SMT mask to see if the entire core is idle, and records this |
| 5705 | * information in sd_llc_shared->has_idle_cores. |
| 5706 | * |
| 5707 | * Since SMT siblings share all cache levels, inspecting this limited remote |
| 5708 | * state should be fairly cheap. |
| 5709 | */ |
Peter Zijlstra | 1b568f0 | 2016-05-09 10:38:41 +0200 | [diff] [blame] | 5710 | void __update_idle_core(struct rq *rq) |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5711 | { |
| 5712 | int core = cpu_of(rq); |
| 5713 | int cpu; |
| 5714 | |
| 5715 | rcu_read_lock(); |
| 5716 | if (test_idle_cores(core, true)) |
| 5717 | goto unlock; |
| 5718 | |
| 5719 | for_each_cpu(cpu, cpu_smt_mask(core)) { |
| 5720 | if (cpu == core) |
| 5721 | continue; |
| 5722 | |
| 5723 | if (!idle_cpu(cpu)) |
| 5724 | goto unlock; |
| 5725 | } |
| 5726 | |
| 5727 | set_idle_cores(core, 1); |
| 5728 | unlock: |
| 5729 | rcu_read_unlock(); |
| 5730 | } |
| 5731 | |
| 5732 | /* |
| 5733 | * Scan the entire LLC domain for idle cores; this dynamically switches off if |
| 5734 | * there are no idle cores left in the system; tracked through |
| 5735 | * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. |
| 5736 | */ |
| 5737 | static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) |
| 5738 | { |
| 5739 | struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); |
Peter Zijlstra | c743f0a | 2017-04-14 14:20:05 +0200 | [diff] [blame] | 5740 | int core, cpu; |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5741 | |
Peter Zijlstra | 1b568f0 | 2016-05-09 10:38:41 +0200 | [diff] [blame] | 5742 | if (!static_branch_likely(&sched_smt_present)) |
| 5743 | return -1; |
| 5744 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5745 | if (!test_idle_cores(target, false)) |
| 5746 | return -1; |
| 5747 | |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 5748 | cpumask_and(cpus, sched_domain_span(sd), &p->cpus_allowed); |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5749 | |
Peter Zijlstra | c743f0a | 2017-04-14 14:20:05 +0200 | [diff] [blame] | 5750 | for_each_cpu_wrap(core, cpus, target) { |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5751 | bool idle = true; |
| 5752 | |
| 5753 | for_each_cpu(cpu, cpu_smt_mask(core)) { |
| 5754 | cpumask_clear_cpu(cpu, cpus); |
| 5755 | if (!idle_cpu(cpu)) |
| 5756 | idle = false; |
| 5757 | } |
| 5758 | |
| 5759 | if (idle) |
| 5760 | return core; |
| 5761 | } |
| 5762 | |
| 5763 | /* |
| 5764 | * Failed to find an idle core; stop looking for one. |
| 5765 | */ |
| 5766 | set_idle_cores(target, 0); |
| 5767 | |
| 5768 | return -1; |
| 5769 | } |
| 5770 | |
| 5771 | /* |
| 5772 | * Scan the local SMT mask for idle CPUs. |
| 5773 | */ |
| 5774 | static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) |
| 5775 | { |
| 5776 | int cpu; |
| 5777 | |
Peter Zijlstra | 1b568f0 | 2016-05-09 10:38:41 +0200 | [diff] [blame] | 5778 | if (!static_branch_likely(&sched_smt_present)) |
| 5779 | return -1; |
| 5780 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5781 | for_each_cpu(cpu, cpu_smt_mask(target)) { |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 5782 | if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5783 | continue; |
| 5784 | if (idle_cpu(cpu)) |
| 5785 | return cpu; |
| 5786 | } |
| 5787 | |
| 5788 | return -1; |
| 5789 | } |
| 5790 | |
| 5791 | #else /* CONFIG_SCHED_SMT */ |
| 5792 | |
| 5793 | static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) |
| 5794 | { |
| 5795 | return -1; |
| 5796 | } |
| 5797 | |
| 5798 | static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) |
| 5799 | { |
| 5800 | return -1; |
| 5801 | } |
| 5802 | |
| 5803 | #endif /* CONFIG_SCHED_SMT */ |
| 5804 | |
| 5805 | /* |
| 5806 | * Scan the LLC domain for idle CPUs; this is dynamically regulated by |
| 5807 | * comparing the average scan cost (tracked in sd->avg_scan_cost) against the |
| 5808 | * average idle time for this rq (as found in rq->avg_idle). |
| 5809 | */ |
| 5810 | static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int target) |
| 5811 | { |
Wanpeng Li | 9cfb38a | 2016-10-09 08:04:03 +0800 | [diff] [blame] | 5812 | struct sched_domain *this_sd; |
Peter Zijlstra | 1ad3aaf | 2017-05-17 12:53:50 +0200 | [diff] [blame] | 5813 | u64 avg_cost, avg_idle; |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5814 | u64 time, cost; |
| 5815 | s64 delta; |
Peter Zijlstra | 1ad3aaf | 2017-05-17 12:53:50 +0200 | [diff] [blame] | 5816 | int cpu, nr = INT_MAX; |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5817 | |
Wanpeng Li | 9cfb38a | 2016-10-09 08:04:03 +0800 | [diff] [blame] | 5818 | this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); |
| 5819 | if (!this_sd) |
| 5820 | return -1; |
| 5821 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5822 | /* |
| 5823 | * Due to large variance we need a large fuzz factor; hackbench in |
| 5824 | * particularly is sensitive here. |
| 5825 | */ |
Peter Zijlstra | 1ad3aaf | 2017-05-17 12:53:50 +0200 | [diff] [blame] | 5826 | avg_idle = this_rq()->avg_idle / 512; |
| 5827 | avg_cost = this_sd->avg_scan_cost + 1; |
| 5828 | |
| 5829 | if (sched_feat(SIS_AVG_CPU) && avg_idle < avg_cost) |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5830 | return -1; |
| 5831 | |
Peter Zijlstra | 1ad3aaf | 2017-05-17 12:53:50 +0200 | [diff] [blame] | 5832 | if (sched_feat(SIS_PROP)) { |
| 5833 | u64 span_avg = sd->span_weight * avg_idle; |
| 5834 | if (span_avg > 4*avg_cost) |
| 5835 | nr = div_u64(span_avg, avg_cost); |
| 5836 | else |
| 5837 | nr = 4; |
| 5838 | } |
| 5839 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5840 | time = local_clock(); |
| 5841 | |
Peter Zijlstra | c743f0a | 2017-04-14 14:20:05 +0200 | [diff] [blame] | 5842 | for_each_cpu_wrap(cpu, sched_domain_span(sd), target) { |
Peter Zijlstra | 1ad3aaf | 2017-05-17 12:53:50 +0200 | [diff] [blame] | 5843 | if (!--nr) |
| 5844 | return -1; |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 5845 | if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5846 | continue; |
| 5847 | if (idle_cpu(cpu)) |
| 5848 | break; |
| 5849 | } |
| 5850 | |
| 5851 | time = local_clock() - time; |
| 5852 | cost = this_sd->avg_scan_cost; |
| 5853 | delta = (s64)(time - cost) / 8; |
| 5854 | this_sd->avg_scan_cost += delta; |
| 5855 | |
| 5856 | return cpu; |
| 5857 | } |
| 5858 | |
| 5859 | /* |
| 5860 | * Try and locate an idle core/thread in the LLC cache domain. |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5861 | */ |
Morten Rasmussen | 772bd008c | 2016-06-22 18:03:13 +0100 | [diff] [blame] | 5862 | static int select_idle_sibling(struct task_struct *p, int prev, int target) |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5863 | { |
Suresh Siddha | 99bd5e2 | 2010-03-31 16:47:45 -0700 | [diff] [blame] | 5864 | struct sched_domain *sd; |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5865 | int i; |
Mike Galbraith | e0a79f5 | 2013-01-28 12:19:25 +0100 | [diff] [blame] | 5866 | |
| 5867 | if (idle_cpu(target)) |
| 5868 | return target; |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5869 | |
| 5870 | /* |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5871 | * If the previous cpu is cache affine and idle, don't be stupid. |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5872 | */ |
Morten Rasmussen | 772bd008c | 2016-06-22 18:03:13 +0100 | [diff] [blame] | 5873 | if (prev != target && cpus_share_cache(prev, target) && idle_cpu(prev)) |
| 5874 | return prev; |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5875 | |
Peter Zijlstra | 518cd62 | 2011-12-07 15:07:31 +0100 | [diff] [blame] | 5876 | sd = rcu_dereference(per_cpu(sd_llc, target)); |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5877 | if (!sd) |
| 5878 | return target; |
Morten Rasmussen | 772bd008c | 2016-06-22 18:03:13 +0100 | [diff] [blame] | 5879 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5880 | i = select_idle_core(p, sd, target); |
| 5881 | if ((unsigned)i < nr_cpumask_bits) |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 5882 | return i; |
Ingo Molnar | 098fb9d | 2008-03-16 20:36:10 +0100 | [diff] [blame] | 5883 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5884 | i = select_idle_cpu(p, sd, target); |
| 5885 | if ((unsigned)i < nr_cpumask_bits) |
| 5886 | return i; |
Mike Galbraith | 970e178 | 2012-06-12 05:18:32 +0200 | [diff] [blame] | 5887 | |
Peter Zijlstra | 10e2f1a | 2016-05-09 10:38:05 +0200 | [diff] [blame] | 5888 | i = select_idle_smt(p, sd, target); |
| 5889 | if ((unsigned)i < nr_cpumask_bits) |
| 5890 | return i; |
Linus Torvalds | 37407ea | 2012-09-16 12:29:43 -0700 | [diff] [blame] | 5891 | |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5892 | return target; |
| 5893 | } |
Dietmar Eggemann | 231678b | 2015-08-14 17:23:13 +0100 | [diff] [blame] | 5894 | |
Vincent Guittot | 8bb5b00 | 2015-03-04 08:48:47 +0100 | [diff] [blame] | 5895 | /* |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 5896 | * cpu_util returns the amount of capacity of a CPU that is used by CFS |
Vincent Guittot | 8bb5b00 | 2015-03-04 08:48:47 +0100 | [diff] [blame] | 5897 | * tasks. The unit of the return value must be the one of capacity so we can |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 5898 | * compare the utilization with the capacity of the CPU that is available for |
| 5899 | * CFS task (ie cpu_capacity). |
Dietmar Eggemann | 231678b | 2015-08-14 17:23:13 +0100 | [diff] [blame] | 5900 | * |
| 5901 | * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the |
| 5902 | * recent utilization of currently non-runnable tasks on a CPU. It represents |
| 5903 | * the amount of utilization of a CPU in the range [0..capacity_orig] where |
| 5904 | * capacity_orig is the cpu_capacity available at the highest frequency |
| 5905 | * (arch_scale_freq_capacity()). |
| 5906 | * The utilization of a CPU converges towards a sum equal to or less than the |
| 5907 | * current capacity (capacity_curr <= capacity_orig) of the CPU because it is |
| 5908 | * the running time on this CPU scaled by capacity_curr. |
| 5909 | * |
| 5910 | * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even |
| 5911 | * higher than capacity_orig because of unfortunate rounding in |
| 5912 | * cfs.avg.util_avg or just after migrating tasks and new task wakeups until |
| 5913 | * the average stabilizes with the new running time. We need to check that the |
| 5914 | * utilization stays within the range of [0..capacity_orig] and cap it if |
| 5915 | * necessary. Without utilization capping, a group could be seen as overloaded |
| 5916 | * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of |
| 5917 | * available capacity. We allow utilization to overshoot capacity_curr (but not |
| 5918 | * capacity_orig) as it useful for predicting the capacity required after task |
| 5919 | * migrations (scheduler-driven DVFS). |
Vincent Guittot | 8bb5b00 | 2015-03-04 08:48:47 +0100 | [diff] [blame] | 5920 | */ |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 5921 | static int cpu_util(int cpu) |
Vincent Guittot | 8bb5b00 | 2015-03-04 08:48:47 +0100 | [diff] [blame] | 5922 | { |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 5923 | unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg; |
Vincent Guittot | 8bb5b00 | 2015-03-04 08:48:47 +0100 | [diff] [blame] | 5924 | unsigned long capacity = capacity_orig_of(cpu); |
| 5925 | |
Dietmar Eggemann | 231678b | 2015-08-14 17:23:13 +0100 | [diff] [blame] | 5926 | return (util >= capacity) ? capacity : util; |
Vincent Guittot | 8bb5b00 | 2015-03-04 08:48:47 +0100 | [diff] [blame] | 5927 | } |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5928 | |
Morten Rasmussen | 3273163 | 2016-07-25 14:34:26 +0100 | [diff] [blame] | 5929 | static inline int task_util(struct task_struct *p) |
| 5930 | { |
| 5931 | return p->se.avg.util_avg; |
| 5932 | } |
| 5933 | |
| 5934 | /* |
Morten Rasmussen | 104cb16 | 2016-10-14 14:41:07 +0100 | [diff] [blame] | 5935 | * cpu_util_wake: Compute cpu utilization with any contributions from |
| 5936 | * the waking task p removed. |
| 5937 | */ |
| 5938 | static int cpu_util_wake(int cpu, struct task_struct *p) |
| 5939 | { |
| 5940 | unsigned long util, capacity; |
| 5941 | |
| 5942 | /* Task has no contribution or is new */ |
| 5943 | if (cpu != task_cpu(p) || !p->se.avg.last_update_time) |
| 5944 | return cpu_util(cpu); |
| 5945 | |
| 5946 | capacity = capacity_orig_of(cpu); |
| 5947 | util = max_t(long, cpu_rq(cpu)->cfs.avg.util_avg - task_util(p), 0); |
| 5948 | |
| 5949 | return (util >= capacity) ? capacity : util; |
| 5950 | } |
| 5951 | |
| 5952 | /* |
Morten Rasmussen | 3273163 | 2016-07-25 14:34:26 +0100 | [diff] [blame] | 5953 | * Disable WAKE_AFFINE in the case where task @p doesn't fit in the |
| 5954 | * capacity of either the waking CPU @cpu or the previous CPU @prev_cpu. |
| 5955 | * |
| 5956 | * In that case WAKE_AFFINE doesn't make sense and we'll let |
| 5957 | * BALANCE_WAKE sort things out. |
| 5958 | */ |
| 5959 | static int wake_cap(struct task_struct *p, int cpu, int prev_cpu) |
| 5960 | { |
| 5961 | long min_cap, max_cap; |
| 5962 | |
| 5963 | min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu)); |
| 5964 | max_cap = cpu_rq(cpu)->rd->max_cpu_capacity; |
| 5965 | |
| 5966 | /* Minimum capacity is close to max, no need to abort wake_affine */ |
| 5967 | if (max_cap - min_cap < max_cap >> 3) |
| 5968 | return 0; |
| 5969 | |
Morten Rasmussen | 104cb16 | 2016-10-14 14:41:07 +0100 | [diff] [blame] | 5970 | /* Bring task utilization in sync with prev_cpu */ |
| 5971 | sync_entity_load_avg(&p->se); |
| 5972 | |
Morten Rasmussen | 3273163 | 2016-07-25 14:34:26 +0100 | [diff] [blame] | 5973 | return min_cap * 1024 < task_util(p) * capacity_margin; |
| 5974 | } |
| 5975 | |
Peter Zijlstra | a50bde5 | 2009-11-12 15:55:28 +0100 | [diff] [blame] | 5976 | /* |
Morten Rasmussen | de91b9c | 2014-02-18 14:14:24 +0000 | [diff] [blame] | 5977 | * select_task_rq_fair: Select target runqueue for the waking task in domains |
| 5978 | * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE, |
| 5979 | * SD_BALANCE_FORK, or SD_BALANCE_EXEC. |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5980 | * |
Morten Rasmussen | de91b9c | 2014-02-18 14:14:24 +0000 | [diff] [blame] | 5981 | * Balances load by selecting the idlest cpu in the idlest group, or under |
| 5982 | * certain conditions an idle sibling cpu if the domain has SD_WAKE_AFFINE set. |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5983 | * |
Morten Rasmussen | de91b9c | 2014-02-18 14:14:24 +0000 | [diff] [blame] | 5984 | * Returns the target cpu number. |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5985 | * |
| 5986 | * preempt must be disabled. |
| 5987 | */ |
Peter Zijlstra | 0017d73 | 2010-03-24 18:34:10 +0100 | [diff] [blame] | 5988 | static int |
Peter Zijlstra | ac66f54 | 2013-10-07 11:29:16 +0100 | [diff] [blame] | 5989 | select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 5990 | { |
Peter Zijlstra | 29cd8ba | 2009-09-17 09:01:14 +0200 | [diff] [blame] | 5991 | struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; |
Peter Zijlstra | c88d591 | 2009-09-10 13:50:02 +0200 | [diff] [blame] | 5992 | int cpu = smp_processor_id(); |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 5993 | int new_cpu = prev_cpu; |
Suresh Siddha | 99bd5e2 | 2010-03-31 16:47:45 -0700 | [diff] [blame] | 5994 | int want_affine = 0; |
Peter Zijlstra | 5158f4e | 2009-09-16 13:46:59 +0200 | [diff] [blame] | 5995 | int sync = wake_flags & WF_SYNC; |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 5996 | |
Peter Zijlstra | c58d25f | 2016-05-12 09:19:59 +0200 | [diff] [blame] | 5997 | if (sd_flag & SD_BALANCE_WAKE) { |
| 5998 | record_wakee(p); |
Morten Rasmussen | 3273163 | 2016-07-25 14:34:26 +0100 | [diff] [blame] | 5999 | want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 6000 | && cpumask_test_cpu(cpu, &p->cpus_allowed); |
Peter Zijlstra | c58d25f | 2016-05-12 09:19:59 +0200 | [diff] [blame] | 6001 | } |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 6002 | |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 6003 | rcu_read_lock(); |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6004 | for_each_domain(cpu, tmp) { |
Peter Zijlstra | e4f4288 | 2009-12-16 18:04:34 +0100 | [diff] [blame] | 6005 | if (!(tmp->flags & SD_LOAD_BALANCE)) |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 6006 | break; |
Peter Zijlstra | e4f4288 | 2009-12-16 18:04:34 +0100 | [diff] [blame] | 6007 | |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6008 | /* |
Suresh Siddha | 99bd5e2 | 2010-03-31 16:47:45 -0700 | [diff] [blame] | 6009 | * If both cpu and prev_cpu are part of this domain, |
| 6010 | * cpu is a valid SD_WAKE_AFFINE target. |
Peter Zijlstra | fe3bcfe | 2009-11-12 15:55:29 +0100 | [diff] [blame] | 6011 | */ |
Suresh Siddha | 99bd5e2 | 2010-03-31 16:47:45 -0700 | [diff] [blame] | 6012 | if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && |
| 6013 | cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { |
| 6014 | affine_sd = tmp; |
Alex Shi | f03542a | 2012-07-26 08:55:34 +0800 | [diff] [blame] | 6015 | break; |
Peter Zijlstra | c88d591 | 2009-09-10 13:50:02 +0200 | [diff] [blame] | 6016 | } |
| 6017 | |
Alex Shi | f03542a | 2012-07-26 08:55:34 +0800 | [diff] [blame] | 6018 | if (tmp->flags & sd_flag) |
Peter Zijlstra | 29cd8ba | 2009-09-17 09:01:14 +0200 | [diff] [blame] | 6019 | sd = tmp; |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 6020 | else if (!want_affine) |
| 6021 | break; |
Peter Zijlstra | c88d591 | 2009-09-10 13:50:02 +0200 | [diff] [blame] | 6022 | } |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6023 | |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 6024 | if (affine_sd) { |
| 6025 | sd = NULL; /* Prefer wake_affine over balance flags */ |
Rik van Riel | 7d894e6 | 2017-06-23 12:55:28 -0400 | [diff] [blame] | 6026 | if (cpu == prev_cpu) |
| 6027 | goto pick_cpu; |
| 6028 | |
| 6029 | if (wake_affine(affine_sd, p, prev_cpu, sync)) |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 6030 | new_cpu = cpu; |
Mike Galbraith | 8b911ac | 2010-03-11 17:17:16 +0100 | [diff] [blame] | 6031 | } |
Peter Zijlstra | 3b64089 | 2009-09-16 13:44:33 +0200 | [diff] [blame] | 6032 | |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 6033 | if (!sd) { |
Rik van Riel | 7d894e6 | 2017-06-23 12:55:28 -0400 | [diff] [blame] | 6034 | pick_cpu: |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 6035 | if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */ |
Morten Rasmussen | 772bd008c | 2016-06-22 18:03:13 +0100 | [diff] [blame] | 6036 | new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); |
Mike Galbraith | 63b0e9e | 2015-07-14 17:39:50 +0200 | [diff] [blame] | 6037 | |
| 6038 | } else while (sd) { |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6039 | struct sched_group *group; |
Peter Zijlstra | c88d591 | 2009-09-10 13:50:02 +0200 | [diff] [blame] | 6040 | int weight; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6041 | |
Peter Zijlstra | 0763a66 | 2009-09-14 19:37:39 +0200 | [diff] [blame] | 6042 | if (!(sd->flags & sd_flag)) { |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6043 | sd = sd->child; |
| 6044 | continue; |
| 6045 | } |
| 6046 | |
Vincent Guittot | c44f2a0 | 2013-10-18 13:52:21 +0200 | [diff] [blame] | 6047 | group = find_idlest_group(sd, p, cpu, sd_flag); |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6048 | if (!group) { |
| 6049 | sd = sd->child; |
| 6050 | continue; |
| 6051 | } |
| 6052 | |
Peter Zijlstra | d7c33c4 | 2009-09-11 12:45:38 +0200 | [diff] [blame] | 6053 | new_cpu = find_idlest_cpu(group, p, cpu); |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6054 | if (new_cpu == -1 || new_cpu == cpu) { |
| 6055 | /* Now try balancing at a lower domain level of cpu */ |
| 6056 | sd = sd->child; |
| 6057 | continue; |
| 6058 | } |
| 6059 | |
| 6060 | /* Now try balancing at a lower domain level of new_cpu */ |
| 6061 | cpu = new_cpu; |
Peter Zijlstra | 669c55e | 2010-04-16 14:59:29 +0200 | [diff] [blame] | 6062 | weight = sd->span_weight; |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6063 | sd = NULL; |
| 6064 | for_each_domain(cpu, tmp) { |
Peter Zijlstra | 669c55e | 2010-04-16 14:59:29 +0200 | [diff] [blame] | 6065 | if (weight <= tmp->span_weight) |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6066 | break; |
Peter Zijlstra | 0763a66 | 2009-09-14 19:37:39 +0200 | [diff] [blame] | 6067 | if (tmp->flags & sd_flag) |
Peter Zijlstra | aaee120 | 2009-09-10 13:36:25 +0200 | [diff] [blame] | 6068 | sd = tmp; |
| 6069 | } |
| 6070 | /* while loop will break here if sd == NULL */ |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 6071 | } |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 6072 | rcu_read_unlock(); |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 6073 | |
Peter Zijlstra | c88d591 | 2009-09-10 13:50:02 +0200 | [diff] [blame] | 6074 | return new_cpu; |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 6075 | } |
Paul Turner | 0a74bef | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 6076 | |
| 6077 | /* |
| 6078 | * Called immediately before a task is migrated to a new cpu; task_cpu(p) and |
| 6079 | * cfs_rq_of(p) references at time of call are still valid and identify the |
Byungchul Park | 525628c | 2015-11-18 09:34:59 +0900 | [diff] [blame] | 6080 | * previous cpu. The caller guarantees p->pi_lock or task_rq(p)->lock is held. |
Paul Turner | 0a74bef | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 6081 | */ |
xiaofeng.yan | 5a4fd03 | 2015-09-23 14:55:59 +0800 | [diff] [blame] | 6082 | static void migrate_task_rq_fair(struct task_struct *p) |
Paul Turner | 0a74bef | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 6083 | { |
Paul Turner | aff3e49 | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 6084 | /* |
Peter Zijlstra | 59efa0b | 2016-05-10 18:24:37 +0200 | [diff] [blame] | 6085 | * As blocked tasks retain absolute vruntime the migration needs to |
| 6086 | * deal with this by subtracting the old and adding the new |
| 6087 | * min_vruntime -- the latter is done by enqueue_entity() when placing |
| 6088 | * the task on the new runqueue. |
| 6089 | */ |
| 6090 | if (p->state == TASK_WAKING) { |
| 6091 | struct sched_entity *se = &p->se; |
| 6092 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 6093 | u64 min_vruntime; |
| 6094 | |
| 6095 | #ifndef CONFIG_64BIT |
| 6096 | u64 min_vruntime_copy; |
| 6097 | |
| 6098 | do { |
| 6099 | min_vruntime_copy = cfs_rq->min_vruntime_copy; |
| 6100 | smp_rmb(); |
| 6101 | min_vruntime = cfs_rq->min_vruntime; |
| 6102 | } while (min_vruntime != min_vruntime_copy); |
| 6103 | #else |
| 6104 | min_vruntime = cfs_rq->min_vruntime; |
| 6105 | #endif |
| 6106 | |
| 6107 | se->vruntime -= min_vruntime; |
| 6108 | } |
| 6109 | |
| 6110 | /* |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 6111 | * We are supposed to update the task to "current" time, then its up to date |
| 6112 | * and ready to go to new CPU/cfs_rq. But we have difficulty in getting |
| 6113 | * what current time is, so simply throw away the out-of-date time. This |
| 6114 | * will result in the wakee task is less decayed, but giving the wakee more |
| 6115 | * load sounds not bad. |
Paul Turner | aff3e49 | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 6116 | */ |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 6117 | remove_entity_load_avg(&p->se); |
| 6118 | |
| 6119 | /* Tell new CPU we are migrated */ |
| 6120 | p->se.avg.last_update_time = 0; |
Ben Segall | 3944a92 | 2014-05-15 15:59:20 -0700 | [diff] [blame] | 6121 | |
| 6122 | /* We have migrated, no longer consider this task hot */ |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 6123 | p->se.exec_start = 0; |
Paul Turner | 0a74bef | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 6124 | } |
Yuyang Du | 1269557 | 2015-07-15 08:04:40 +0800 | [diff] [blame] | 6125 | |
| 6126 | static void task_dead_fair(struct task_struct *p) |
| 6127 | { |
| 6128 | remove_entity_load_avg(&p->se); |
| 6129 | } |
Gregory Haskins | e7693a3 | 2008-01-25 21:08:09 +0100 | [diff] [blame] | 6130 | #endif /* CONFIG_SMP */ |
| 6131 | |
Peter Zijlstra | e52fb7c | 2009-01-14 12:39:19 +0100 | [diff] [blame] | 6132 | static unsigned long |
| 6133 | wakeup_gran(struct sched_entity *curr, struct sched_entity *se) |
Peter Zijlstra | 0bbd333 | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 6134 | { |
| 6135 | unsigned long gran = sysctl_sched_wakeup_granularity; |
| 6136 | |
| 6137 | /* |
Peter Zijlstra | e52fb7c | 2009-01-14 12:39:19 +0100 | [diff] [blame] | 6138 | * Since its curr running now, convert the gran from real-time |
| 6139 | * to virtual-time in his units. |
Mike Galbraith | 13814d4 | 2010-03-11 17:17:04 +0100 | [diff] [blame] | 6140 | * |
| 6141 | * By using 'se' instead of 'curr' we penalize light tasks, so |
| 6142 | * they get preempted easier. That is, if 'se' < 'curr' then |
| 6143 | * the resulting gran will be larger, therefore penalizing the |
| 6144 | * lighter, if otoh 'se' > 'curr' then the resulting gran will |
| 6145 | * be smaller, again penalizing the lighter task. |
| 6146 | * |
| 6147 | * This is especially important for buddies when the leftmost |
| 6148 | * task is higher priority than the buddy. |
Peter Zijlstra | 0bbd333 | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 6149 | */ |
Shaohua Li | f4ad9bd | 2011-04-08 12:53:09 +0800 | [diff] [blame] | 6150 | return calc_delta_fair(gran, se); |
Peter Zijlstra | 0bbd333 | 2008-04-19 19:44:57 +0200 | [diff] [blame] | 6151 | } |
| 6152 | |
| 6153 | /* |
Peter Zijlstra | 464b752 | 2008-10-24 11:06:15 +0200 | [diff] [blame] | 6154 | * Should 'se' preempt 'curr'. |
| 6155 | * |
| 6156 | * |s1 |
| 6157 | * |s2 |
| 6158 | * |s3 |
| 6159 | * g |
| 6160 | * |<--->|c |
| 6161 | * |
| 6162 | * w(c, s1) = -1 |
| 6163 | * w(c, s2) = 0 |
| 6164 | * w(c, s3) = 1 |
| 6165 | * |
| 6166 | */ |
| 6167 | static int |
| 6168 | wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) |
| 6169 | { |
| 6170 | s64 gran, vdiff = curr->vruntime - se->vruntime; |
| 6171 | |
| 6172 | if (vdiff <= 0) |
| 6173 | return -1; |
| 6174 | |
Peter Zijlstra | e52fb7c | 2009-01-14 12:39:19 +0100 | [diff] [blame] | 6175 | gran = wakeup_gran(curr, se); |
Peter Zijlstra | 464b752 | 2008-10-24 11:06:15 +0200 | [diff] [blame] | 6176 | if (vdiff > gran) |
| 6177 | return 1; |
| 6178 | |
| 6179 | return 0; |
| 6180 | } |
| 6181 | |
Peter Zijlstra | 0247909 | 2008-11-04 21:25:10 +0100 | [diff] [blame] | 6182 | static void set_last_buddy(struct sched_entity *se) |
| 6183 | { |
Venkatesh Pallipadi | 69c80f3 | 2011-04-13 18:21:09 -0700 | [diff] [blame] | 6184 | if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE)) |
| 6185 | return; |
| 6186 | |
Daniel Axtens | c5ae366 | 2017-05-11 06:11:39 +1000 | [diff] [blame] | 6187 | for_each_sched_entity(se) { |
| 6188 | if (SCHED_WARN_ON(!se->on_rq)) |
| 6189 | return; |
Venkatesh Pallipadi | 69c80f3 | 2011-04-13 18:21:09 -0700 | [diff] [blame] | 6190 | cfs_rq_of(se)->last = se; |
Daniel Axtens | c5ae366 | 2017-05-11 06:11:39 +1000 | [diff] [blame] | 6191 | } |
Peter Zijlstra | 0247909 | 2008-11-04 21:25:10 +0100 | [diff] [blame] | 6192 | } |
| 6193 | |
| 6194 | static void set_next_buddy(struct sched_entity *se) |
| 6195 | { |
Venkatesh Pallipadi | 69c80f3 | 2011-04-13 18:21:09 -0700 | [diff] [blame] | 6196 | if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE)) |
| 6197 | return; |
| 6198 | |
Daniel Axtens | c5ae366 | 2017-05-11 06:11:39 +1000 | [diff] [blame] | 6199 | for_each_sched_entity(se) { |
| 6200 | if (SCHED_WARN_ON(!se->on_rq)) |
| 6201 | return; |
Venkatesh Pallipadi | 69c80f3 | 2011-04-13 18:21:09 -0700 | [diff] [blame] | 6202 | cfs_rq_of(se)->next = se; |
Daniel Axtens | c5ae366 | 2017-05-11 06:11:39 +1000 | [diff] [blame] | 6203 | } |
Peter Zijlstra | 0247909 | 2008-11-04 21:25:10 +0100 | [diff] [blame] | 6204 | } |
| 6205 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 6206 | static void set_skip_buddy(struct sched_entity *se) |
| 6207 | { |
Venkatesh Pallipadi | 69c80f3 | 2011-04-13 18:21:09 -0700 | [diff] [blame] | 6208 | for_each_sched_entity(se) |
| 6209 | cfs_rq_of(se)->skip = se; |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 6210 | } |
| 6211 | |
Peter Zijlstra | 464b752 | 2008-10-24 11:06:15 +0200 | [diff] [blame] | 6212 | /* |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6213 | * Preempt the current task with a newly woken task if needed: |
| 6214 | */ |
Peter Zijlstra | 5a9b86f | 2009-09-16 13:47:58 +0200 | [diff] [blame] | 6215 | static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6216 | { |
| 6217 | struct task_struct *curr = rq->curr; |
Srivatsa Vaddagiri | 8651a86 | 2007-10-15 17:00:12 +0200 | [diff] [blame] | 6218 | struct sched_entity *se = &curr->se, *pse = &p->se; |
Mike Galbraith | 03e89e4 | 2008-12-16 08:45:30 +0100 | [diff] [blame] | 6219 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); |
Mike Galbraith | f685cea | 2009-10-23 23:09:22 +0200 | [diff] [blame] | 6220 | int scale = cfs_rq->nr_running >= sched_nr_latency; |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 6221 | int next_buddy_marked = 0; |
Mike Galbraith | 03e89e4 | 2008-12-16 08:45:30 +0100 | [diff] [blame] | 6222 | |
Ingo Molnar | 4ae7d5c | 2008-03-19 01:42:00 +0100 | [diff] [blame] | 6223 | if (unlikely(se == pse)) |
| 6224 | return; |
| 6225 | |
Paul Turner | 5238cdd | 2011-07-21 09:43:37 -0700 | [diff] [blame] | 6226 | /* |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6227 | * This is possible from callers such as attach_tasks(), in which we |
Paul Turner | 5238cdd | 2011-07-21 09:43:37 -0700 | [diff] [blame] | 6228 | * unconditionally check_prempt_curr() after an enqueue (which may have |
| 6229 | * lead to a throttle). This both saves work and prevents false |
| 6230 | * next-buddy nomination below. |
| 6231 | */ |
| 6232 | if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) |
| 6233 | return; |
| 6234 | |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 6235 | if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { |
Mike Galbraith | 3cb63d5 | 2009-09-11 12:01:17 +0200 | [diff] [blame] | 6236 | set_next_buddy(pse); |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 6237 | next_buddy_marked = 1; |
| 6238 | } |
Peter Zijlstra | 57fdc26 | 2008-09-23 15:33:45 +0200 | [diff] [blame] | 6239 | |
Bharata B Rao | aec0a51 | 2008-08-28 14:42:49 +0530 | [diff] [blame] | 6240 | /* |
| 6241 | * We can come here with TIF_NEED_RESCHED already set from new task |
| 6242 | * wake up path. |
Paul Turner | 5238cdd | 2011-07-21 09:43:37 -0700 | [diff] [blame] | 6243 | * |
| 6244 | * Note: this also catches the edge-case of curr being in a throttled |
| 6245 | * group (e.g. via set_curr_task), since update_curr() (in the |
| 6246 | * enqueue of curr) will have resulted in resched being set. This |
| 6247 | * prevents us from potentially nominating it as a false LAST_BUDDY |
| 6248 | * below. |
Bharata B Rao | aec0a51 | 2008-08-28 14:42:49 +0530 | [diff] [blame] | 6249 | */ |
| 6250 | if (test_tsk_need_resched(curr)) |
| 6251 | return; |
| 6252 | |
Darren Hart | a2f5c9a | 2011-02-22 13:04:33 -0800 | [diff] [blame] | 6253 | /* Idle tasks are by definition preempted by non-idle tasks. */ |
| 6254 | if (unlikely(curr->policy == SCHED_IDLE) && |
| 6255 | likely(p->policy != SCHED_IDLE)) |
| 6256 | goto preempt; |
| 6257 | |
Ingo Molnar | 91c234b | 2007-10-15 17:00:18 +0200 | [diff] [blame] | 6258 | /* |
Darren Hart | a2f5c9a | 2011-02-22 13:04:33 -0800 | [diff] [blame] | 6259 | * Batch and idle tasks do not preempt non-idle tasks (their preemption |
| 6260 | * is driven by the tick): |
Ingo Molnar | 91c234b | 2007-10-15 17:00:18 +0200 | [diff] [blame] | 6261 | */ |
Ingo Molnar | 8ed92e5 | 2012-10-14 14:28:50 +0200 | [diff] [blame] | 6262 | if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) |
Ingo Molnar | 91c234b | 2007-10-15 17:00:18 +0200 | [diff] [blame] | 6263 | return; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6264 | |
Peter Zijlstra | 3a7e73a | 2009-11-28 18:51:02 +0100 | [diff] [blame] | 6265 | find_matching_se(&se, &pse); |
Paul Turner | 9bbd737 | 2011-07-05 19:07:21 -0700 | [diff] [blame] | 6266 | update_curr(cfs_rq_of(se)); |
Peter Zijlstra | 3a7e73a | 2009-11-28 18:51:02 +0100 | [diff] [blame] | 6267 | BUG_ON(!pse); |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 6268 | if (wakeup_preempt_entity(se, pse) == 1) { |
| 6269 | /* |
| 6270 | * Bias pick_next to pick the sched entity that is |
| 6271 | * triggering this preemption. |
| 6272 | */ |
| 6273 | if (!next_buddy_marked) |
| 6274 | set_next_buddy(pse); |
Peter Zijlstra | 3a7e73a | 2009-11-28 18:51:02 +0100 | [diff] [blame] | 6275 | goto preempt; |
Venkatesh Pallipadi | 2f36825 | 2011-04-14 10:30:53 -0700 | [diff] [blame] | 6276 | } |
Jupyung Lee | a65ac74 | 2009-11-17 18:51:40 +0900 | [diff] [blame] | 6277 | |
Peter Zijlstra | 3a7e73a | 2009-11-28 18:51:02 +0100 | [diff] [blame] | 6278 | return; |
| 6279 | |
| 6280 | preempt: |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 6281 | resched_curr(rq); |
Peter Zijlstra | 3a7e73a | 2009-11-28 18:51:02 +0100 | [diff] [blame] | 6282 | /* |
| 6283 | * Only set the backward buddy when the current task is still |
| 6284 | * on the rq. This can happen when a wakeup gets interleaved |
| 6285 | * with schedule on the ->pre_schedule() or idle_balance() |
| 6286 | * point, either of which can * drop the rq lock. |
| 6287 | * |
| 6288 | * Also, during early boot the idle thread is in the fair class, |
| 6289 | * for obvious reasons its a bad idea to schedule back to it. |
| 6290 | */ |
| 6291 | if (unlikely(!se->on_rq || curr == rq->idle)) |
| 6292 | return; |
| 6293 | |
| 6294 | if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) |
| 6295 | set_last_buddy(se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6296 | } |
| 6297 | |
Peter Zijlstra | 606dba2 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6298 | static struct task_struct * |
Matt Fleming | d8ac897 | 2016-09-21 14:38:10 +0100 | [diff] [blame] | 6299 | pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6300 | { |
| 6301 | struct cfs_rq *cfs_rq = &rq->cfs; |
| 6302 | struct sched_entity *se; |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6303 | struct task_struct *p; |
Peter Zijlstra | 37e117c | 2014-02-14 12:25:08 +0100 | [diff] [blame] | 6304 | int new_tasks; |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6305 | |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 6306 | again: |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6307 | if (!cfs_rq->nr_running) |
Peter Zijlstra | 38033c3 | 2014-01-23 20:32:21 +0100 | [diff] [blame] | 6308 | goto idle; |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6309 | |
Viresh Kumar | 9674f5c | 2017-05-24 10:59:55 +0530 | [diff] [blame] | 6310 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Peter Zijlstra | 3f1d2a3 | 2014-02-12 10:49:30 +0100 | [diff] [blame] | 6311 | if (prev->sched_class != &fair_sched_class) |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6312 | goto simple; |
| 6313 | |
| 6314 | /* |
| 6315 | * Because of the set_next_buddy() in dequeue_task_fair() it is rather |
| 6316 | * likely that a next task is from the same cgroup as the current. |
| 6317 | * |
| 6318 | * Therefore attempt to avoid putting and setting the entire cgroup |
| 6319 | * hierarchy, only change the part that actually changes. |
| 6320 | */ |
| 6321 | |
| 6322 | do { |
| 6323 | struct sched_entity *curr = cfs_rq->curr; |
| 6324 | |
| 6325 | /* |
| 6326 | * Since we got here without doing put_prev_entity() we also |
| 6327 | * have to consider cfs_rq->curr. If it is still a runnable |
| 6328 | * entity, update_curr() will update its vruntime, otherwise |
| 6329 | * forget we've ever seen it. |
| 6330 | */ |
Ben Segall | 54d2736 | 2015-04-06 15:28:10 -0700 | [diff] [blame] | 6331 | if (curr) { |
| 6332 | if (curr->on_rq) |
| 6333 | update_curr(cfs_rq); |
| 6334 | else |
| 6335 | curr = NULL; |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6336 | |
Ben Segall | 54d2736 | 2015-04-06 15:28:10 -0700 | [diff] [blame] | 6337 | /* |
| 6338 | * This call to check_cfs_rq_runtime() will do the |
| 6339 | * throttle and dequeue its entity in the parent(s). |
Viresh Kumar | 9674f5c | 2017-05-24 10:59:55 +0530 | [diff] [blame] | 6340 | * Therefore the nr_running test will indeed |
Ben Segall | 54d2736 | 2015-04-06 15:28:10 -0700 | [diff] [blame] | 6341 | * be correct. |
| 6342 | */ |
Viresh Kumar | 9674f5c | 2017-05-24 10:59:55 +0530 | [diff] [blame] | 6343 | if (unlikely(check_cfs_rq_runtime(cfs_rq))) { |
| 6344 | cfs_rq = &rq->cfs; |
| 6345 | |
| 6346 | if (!cfs_rq->nr_running) |
| 6347 | goto idle; |
| 6348 | |
Ben Segall | 54d2736 | 2015-04-06 15:28:10 -0700 | [diff] [blame] | 6349 | goto simple; |
Viresh Kumar | 9674f5c | 2017-05-24 10:59:55 +0530 | [diff] [blame] | 6350 | } |
Ben Segall | 54d2736 | 2015-04-06 15:28:10 -0700 | [diff] [blame] | 6351 | } |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6352 | |
| 6353 | se = pick_next_entity(cfs_rq, curr); |
| 6354 | cfs_rq = group_cfs_rq(se); |
| 6355 | } while (cfs_rq); |
| 6356 | |
| 6357 | p = task_of(se); |
| 6358 | |
| 6359 | /* |
| 6360 | * Since we haven't yet done put_prev_entity and if the selected task |
| 6361 | * is a different task than we started out with, try and touch the |
| 6362 | * least amount of cfs_rqs. |
| 6363 | */ |
| 6364 | if (prev != p) { |
| 6365 | struct sched_entity *pse = &prev->se; |
| 6366 | |
| 6367 | while (!(cfs_rq = is_same_group(se, pse))) { |
| 6368 | int se_depth = se->depth; |
| 6369 | int pse_depth = pse->depth; |
| 6370 | |
| 6371 | if (se_depth <= pse_depth) { |
| 6372 | put_prev_entity(cfs_rq_of(pse), pse); |
| 6373 | pse = parent_entity(pse); |
| 6374 | } |
| 6375 | if (se_depth >= pse_depth) { |
| 6376 | set_next_entity(cfs_rq_of(se), se); |
| 6377 | se = parent_entity(se); |
| 6378 | } |
| 6379 | } |
| 6380 | |
| 6381 | put_prev_entity(cfs_rq, pse); |
| 6382 | set_next_entity(cfs_rq, se); |
| 6383 | } |
| 6384 | |
| 6385 | if (hrtick_enabled(rq)) |
| 6386 | hrtick_start_fair(rq, p); |
| 6387 | |
| 6388 | return p; |
| 6389 | simple: |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6390 | #endif |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6391 | |
Peter Zijlstra | 3f1d2a3 | 2014-02-12 10:49:30 +0100 | [diff] [blame] | 6392 | put_prev_task(rq, prev); |
Peter Zijlstra | 606dba2 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6393 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6394 | do { |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6395 | se = pick_next_entity(cfs_rq, NULL); |
Peter Zijlstra | f4b6755 | 2008-11-04 21:25:07 +0100 | [diff] [blame] | 6396 | set_next_entity(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6397 | cfs_rq = group_cfs_rq(se); |
| 6398 | } while (cfs_rq); |
| 6399 | |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 6400 | p = task_of(se); |
Peter Zijlstra | 678d571 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 6401 | |
Mike Galbraith | b39e66e | 2011-11-22 15:20:07 +0100 | [diff] [blame] | 6402 | if (hrtick_enabled(rq)) |
| 6403 | hrtick_start_fair(rq, p); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 6404 | |
| 6405 | return p; |
Peter Zijlstra | 38033c3 | 2014-01-23 20:32:21 +0100 | [diff] [blame] | 6406 | |
| 6407 | idle: |
Matt Fleming | 46f69fa | 2016-09-21 14:38:12 +0100 | [diff] [blame] | 6408 | new_tasks = idle_balance(rq, rf); |
| 6409 | |
Peter Zijlstra | 37e117c | 2014-02-14 12:25:08 +0100 | [diff] [blame] | 6410 | /* |
| 6411 | * Because idle_balance() releases (and re-acquires) rq->lock, it is |
| 6412 | * possible for any higher priority task to appear. In that case we |
| 6413 | * must re-start the pick_next_entity() loop. |
| 6414 | */ |
Kirill Tkhai | e4aa358 | 2014-03-06 13:31:55 +0400 | [diff] [blame] | 6415 | if (new_tasks < 0) |
Peter Zijlstra | 37e117c | 2014-02-14 12:25:08 +0100 | [diff] [blame] | 6416 | return RETRY_TASK; |
| 6417 | |
Kirill Tkhai | e4aa358 | 2014-03-06 13:31:55 +0400 | [diff] [blame] | 6418 | if (new_tasks > 0) |
Peter Zijlstra | 38033c3 | 2014-01-23 20:32:21 +0100 | [diff] [blame] | 6419 | goto again; |
Peter Zijlstra | 38033c3 | 2014-01-23 20:32:21 +0100 | [diff] [blame] | 6420 | |
| 6421 | return NULL; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6422 | } |
| 6423 | |
| 6424 | /* |
| 6425 | * Account for a descheduled task: |
| 6426 | */ |
Ingo Molnar | 31ee529 | 2007-08-09 11:16:49 +0200 | [diff] [blame] | 6427 | static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6428 | { |
| 6429 | struct sched_entity *se = &prev->se; |
| 6430 | struct cfs_rq *cfs_rq; |
| 6431 | |
| 6432 | for_each_sched_entity(se) { |
| 6433 | cfs_rq = cfs_rq_of(se); |
Ingo Molnar | ab6cde2 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 6434 | put_prev_entity(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6435 | } |
| 6436 | } |
| 6437 | |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 6438 | /* |
| 6439 | * sched_yield() is very simple |
| 6440 | * |
| 6441 | * The magic of dealing with the ->skip buddy is in pick_next_entity. |
| 6442 | */ |
| 6443 | static void yield_task_fair(struct rq *rq) |
| 6444 | { |
| 6445 | struct task_struct *curr = rq->curr; |
| 6446 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); |
| 6447 | struct sched_entity *se = &curr->se; |
| 6448 | |
| 6449 | /* |
| 6450 | * Are we the only task in the tree? |
| 6451 | */ |
| 6452 | if (unlikely(rq->nr_running == 1)) |
| 6453 | return; |
| 6454 | |
| 6455 | clear_buddies(cfs_rq, se); |
| 6456 | |
| 6457 | if (curr->policy != SCHED_BATCH) { |
| 6458 | update_rq_clock(rq); |
| 6459 | /* |
| 6460 | * Update run-time statistics of the 'current'. |
| 6461 | */ |
| 6462 | update_curr(cfs_rq); |
Mike Galbraith | 916671c | 2011-11-22 15:21:26 +0100 | [diff] [blame] | 6463 | /* |
| 6464 | * Tell update_rq_clock() that we've just updated, |
| 6465 | * so we don't do microscopic update in schedule() |
| 6466 | * and double the fastpath cost. |
| 6467 | */ |
Peter Zijlstra | 9edfbfe | 2015-01-05 11:18:11 +0100 | [diff] [blame] | 6468 | rq_clock_skip_update(rq, true); |
Rik van Riel | ac53db5 | 2011-02-01 09:51:03 -0500 | [diff] [blame] | 6469 | } |
| 6470 | |
| 6471 | set_skip_buddy(se); |
| 6472 | } |
| 6473 | |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 6474 | static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preempt) |
| 6475 | { |
| 6476 | struct sched_entity *se = &p->se; |
| 6477 | |
Paul Turner | 5238cdd | 2011-07-21 09:43:37 -0700 | [diff] [blame] | 6478 | /* throttled hierarchies are not runnable */ |
| 6479 | if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 6480 | return false; |
| 6481 | |
| 6482 | /* Tell the scheduler that we'd really like pse to run next. */ |
| 6483 | set_next_buddy(se); |
| 6484 | |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 6485 | yield_task_fair(rq); |
| 6486 | |
| 6487 | return true; |
| 6488 | } |
| 6489 | |
Peter Williams | 681f3e6 | 2007-10-24 18:23:51 +0200 | [diff] [blame] | 6490 | #ifdef CONFIG_SMP |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6491 | /************************************************** |
Peter Zijlstra | e9c84cb | 2012-07-03 13:53:26 +0200 | [diff] [blame] | 6492 | * Fair scheduling class load-balancing methods. |
| 6493 | * |
| 6494 | * BASICS |
| 6495 | * |
| 6496 | * The purpose of load-balancing is to achieve the same basic fairness the |
| 6497 | * per-cpu scheduler provides, namely provide a proportional amount of compute |
| 6498 | * time to each task. This is expressed in the following equation: |
| 6499 | * |
| 6500 | * W_i,n/P_i == W_j,n/P_j for all i,j (1) |
| 6501 | * |
| 6502 | * Where W_i,n is the n-th weight average for cpu i. The instantaneous weight |
| 6503 | * W_i,0 is defined as: |
| 6504 | * |
| 6505 | * W_i,0 = \Sum_j w_i,j (2) |
| 6506 | * |
| 6507 | * Where w_i,j is the weight of the j-th runnable task on cpu i. This weight |
Yuyang Du | 1c3de5e | 2016-03-30 07:07:51 +0800 | [diff] [blame] | 6508 | * is derived from the nice value as per sched_prio_to_weight[]. |
Peter Zijlstra | e9c84cb | 2012-07-03 13:53:26 +0200 | [diff] [blame] | 6509 | * |
| 6510 | * The weight average is an exponential decay average of the instantaneous |
| 6511 | * weight: |
| 6512 | * |
| 6513 | * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) |
| 6514 | * |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 6515 | * C_i is the compute capacity of cpu i, typically it is the |
Peter Zijlstra | e9c84cb | 2012-07-03 13:53:26 +0200 | [diff] [blame] | 6516 | * fraction of 'recent' time available for SCHED_OTHER task execution. But it |
| 6517 | * can also include other factors [XXX]. |
| 6518 | * |
| 6519 | * To achieve this balance we define a measure of imbalance which follows |
| 6520 | * directly from (1): |
| 6521 | * |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 6522 | * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) |
Peter Zijlstra | e9c84cb | 2012-07-03 13:53:26 +0200 | [diff] [blame] | 6523 | * |
| 6524 | * We them move tasks around to minimize the imbalance. In the continuous |
| 6525 | * function space it is obvious this converges, in the discrete case we get |
| 6526 | * a few fun cases generally called infeasible weight scenarios. |
| 6527 | * |
| 6528 | * [XXX expand on: |
| 6529 | * - infeasible weights; |
| 6530 | * - local vs global optima in the discrete case. ] |
| 6531 | * |
| 6532 | * |
| 6533 | * SCHED DOMAINS |
| 6534 | * |
| 6535 | * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) |
| 6536 | * for all i,j solution, we create a tree of cpus that follows the hardware |
| 6537 | * topology where each level pairs two lower groups (or better). This results |
| 6538 | * in O(log n) layers. Furthermore we reduce the number of cpus going up the |
| 6539 | * tree to only the first of the previous level and we decrease the frequency |
| 6540 | * of load-balance at each level inv. proportional to the number of cpus in |
| 6541 | * the groups. |
| 6542 | * |
| 6543 | * This yields: |
| 6544 | * |
| 6545 | * log_2 n 1 n |
| 6546 | * \Sum { --- * --- * 2^i } = O(n) (5) |
| 6547 | * i = 0 2^i 2^i |
| 6548 | * `- size of each group |
| 6549 | * | | `- number of cpus doing load-balance |
| 6550 | * | `- freq |
| 6551 | * `- sum over all levels |
| 6552 | * |
| 6553 | * Coupled with a limit on how many tasks we can migrate every balance pass, |
| 6554 | * this makes (5) the runtime complexity of the balancer. |
| 6555 | * |
| 6556 | * An important property here is that each CPU is still (indirectly) connected |
| 6557 | * to every other cpu in at most O(log n) steps: |
| 6558 | * |
| 6559 | * The adjacency matrix of the resulting graph is given by: |
| 6560 | * |
Byungchul Park | 97a7142 | 2015-07-05 18:33:48 +0900 | [diff] [blame] | 6561 | * log_2 n |
Peter Zijlstra | e9c84cb | 2012-07-03 13:53:26 +0200 | [diff] [blame] | 6562 | * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) |
| 6563 | * k = 0 |
| 6564 | * |
| 6565 | * And you'll find that: |
| 6566 | * |
| 6567 | * A^(log_2 n)_i,j != 0 for all i,j (7) |
| 6568 | * |
| 6569 | * Showing there's indeed a path between every cpu in at most O(log n) steps. |
| 6570 | * The task movement gives a factor of O(m), giving a convergence complexity |
| 6571 | * of: |
| 6572 | * |
| 6573 | * O(nm log n), n := nr_cpus, m := nr_tasks (8) |
| 6574 | * |
| 6575 | * |
| 6576 | * WORK CONSERVING |
| 6577 | * |
| 6578 | * In order to avoid CPUs going idle while there's still work to do, new idle |
| 6579 | * balancing is more aggressive and has the newly idle cpu iterate up the domain |
| 6580 | * tree itself instead of relying on other CPUs to bring it work. |
| 6581 | * |
| 6582 | * This adds some complexity to both (5) and (8) but it reduces the total idle |
| 6583 | * time. |
| 6584 | * |
| 6585 | * [XXX more?] |
| 6586 | * |
| 6587 | * |
| 6588 | * CGROUPS |
| 6589 | * |
| 6590 | * Cgroups make a horror show out of (2), instead of a simple sum we get: |
| 6591 | * |
| 6592 | * s_k,i |
| 6593 | * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) |
| 6594 | * S_k |
| 6595 | * |
| 6596 | * Where |
| 6597 | * |
| 6598 | * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) |
| 6599 | * |
| 6600 | * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on cpu i. |
| 6601 | * |
| 6602 | * The big problem is S_k, its a global sum needed to compute a local (W_i) |
| 6603 | * property. |
| 6604 | * |
| 6605 | * [XXX write more on how we solve this.. _after_ merging pjt's patches that |
| 6606 | * rewrite all of this once again.] |
Byungchul Park | 97a7142 | 2015-07-05 18:33:48 +0900 | [diff] [blame] | 6607 | */ |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 6608 | |
Hiroshi Shimamoto | ed387b7 | 2012-01-31 11:40:32 +0900 | [diff] [blame] | 6609 | static unsigned long __read_mostly max_load_balance_interval = HZ/10; |
| 6610 | |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 6611 | enum fbq_type { regular, remote, all }; |
| 6612 | |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6613 | #define LBF_ALL_PINNED 0x01 |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6614 | #define LBF_NEED_BREAK 0x02 |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 6615 | #define LBF_DST_PINNED 0x04 |
| 6616 | #define LBF_SOME_PINNED 0x08 |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6617 | |
| 6618 | struct lb_env { |
| 6619 | struct sched_domain *sd; |
| 6620 | |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6621 | struct rq *src_rq; |
Prashanth Nageshappa | 85c1e7d | 2012-06-19 17:47:34 +0530 | [diff] [blame] | 6622 | int src_cpu; |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6623 | |
| 6624 | int dst_cpu; |
| 6625 | struct rq *dst_rq; |
| 6626 | |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6627 | struct cpumask *dst_grpmask; |
| 6628 | int new_dst_cpu; |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6629 | enum cpu_idle_type idle; |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 6630 | long imbalance; |
Michael Wang | b9403130 | 2012-07-12 16:10:13 +0800 | [diff] [blame] | 6631 | /* The set of CPUs under consideration for load-balancing */ |
| 6632 | struct cpumask *cpus; |
| 6633 | |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6634 | unsigned int flags; |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6635 | |
| 6636 | unsigned int loop; |
| 6637 | unsigned int loop_break; |
| 6638 | unsigned int loop_max; |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 6639 | |
| 6640 | enum fbq_type fbq_type; |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6641 | struct list_head tasks; |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6642 | }; |
| 6643 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6644 | /* |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 6645 | * Is this task likely cache-hot: |
| 6646 | */ |
Hillf Danton | 5d5e2b1 | 2014-06-10 10:58:43 +0200 | [diff] [blame] | 6647 | static int task_hot(struct task_struct *p, struct lb_env *env) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 6648 | { |
| 6649 | s64 delta; |
| 6650 | |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6651 | lockdep_assert_held(&env->src_rq->lock); |
| 6652 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 6653 | if (p->sched_class != &fair_sched_class) |
| 6654 | return 0; |
| 6655 | |
| 6656 | if (unlikely(p->policy == SCHED_IDLE)) |
| 6657 | return 0; |
| 6658 | |
| 6659 | /* |
| 6660 | * Buddy candidates are cache hot: |
| 6661 | */ |
Hillf Danton | 5d5e2b1 | 2014-06-10 10:58:43 +0200 | [diff] [blame] | 6662 | if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 6663 | (&p->se == cfs_rq_of(&p->se)->next || |
| 6664 | &p->se == cfs_rq_of(&p->se)->last)) |
| 6665 | return 1; |
| 6666 | |
| 6667 | if (sysctl_sched_migration_cost == -1) |
| 6668 | return 1; |
| 6669 | if (sysctl_sched_migration_cost == 0) |
| 6670 | return 0; |
| 6671 | |
Hillf Danton | 5d5e2b1 | 2014-06-10 10:58:43 +0200 | [diff] [blame] | 6672 | delta = rq_clock_task(env->src_rq) - p->se.exec_start; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 6673 | |
| 6674 | return delta < (s64)sysctl_sched_migration_cost; |
| 6675 | } |
| 6676 | |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6677 | #ifdef CONFIG_NUMA_BALANCING |
Rik van Riel | c1ceac6 | 2015-05-14 22:59:36 -0400 | [diff] [blame] | 6678 | /* |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6679 | * Returns 1, if task migration degrades locality |
| 6680 | * Returns 0, if task migration improves locality i.e migration preferred. |
| 6681 | * Returns -1, if task migration is not affected by locality. |
Rik van Riel | c1ceac6 | 2015-05-14 22:59:36 -0400 | [diff] [blame] | 6682 | */ |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6683 | static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env) |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6684 | { |
Rik van Riel | b1ad065 | 2014-05-15 13:03:06 -0400 | [diff] [blame] | 6685 | struct numa_group *numa_group = rcu_dereference(p->numa_group); |
Rik van Riel | c1ceac6 | 2015-05-14 22:59:36 -0400 | [diff] [blame] | 6686 | unsigned long src_faults, dst_faults; |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6687 | int src_nid, dst_nid; |
| 6688 | |
Srikar Dronamraju | 2a59572 | 2015-08-11 21:54:21 +0530 | [diff] [blame] | 6689 | if (!static_branch_likely(&sched_numa_balancing)) |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6690 | return -1; |
| 6691 | |
Srikar Dronamraju | c3b9bc5 | 2015-08-11 16:30:12 +0530 | [diff] [blame] | 6692 | if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6693 | return -1; |
Mel Gorman | 7a0f308 | 2013-10-07 11:29:01 +0100 | [diff] [blame] | 6694 | |
| 6695 | src_nid = cpu_to_node(env->src_cpu); |
| 6696 | dst_nid = cpu_to_node(env->dst_cpu); |
| 6697 | |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 6698 | if (src_nid == dst_nid) |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6699 | return -1; |
Mel Gorman | 7a0f308 | 2013-10-07 11:29:01 +0100 | [diff] [blame] | 6700 | |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6701 | /* Migrating away from the preferred node is always bad. */ |
| 6702 | if (src_nid == p->numa_preferred_nid) { |
| 6703 | if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) |
| 6704 | return 1; |
| 6705 | else |
| 6706 | return -1; |
| 6707 | } |
Mel Gorman | 83e1d2c | 2013-10-07 11:29:27 +0100 | [diff] [blame] | 6708 | |
Rik van Riel | c1ceac6 | 2015-05-14 22:59:36 -0400 | [diff] [blame] | 6709 | /* Encourage migration to the preferred node. */ |
| 6710 | if (dst_nid == p->numa_preferred_nid) |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6711 | return 0; |
Rik van Riel | c1ceac6 | 2015-05-14 22:59:36 -0400 | [diff] [blame] | 6712 | |
Rik van Riel | 739294f | 2017-06-23 12:55:27 -0400 | [diff] [blame] | 6713 | /* Leaving a core idle is often worse than degrading locality. */ |
| 6714 | if (env->idle != CPU_NOT_IDLE) |
| 6715 | return -1; |
| 6716 | |
Rik van Riel | c1ceac6 | 2015-05-14 22:59:36 -0400 | [diff] [blame] | 6717 | if (numa_group) { |
| 6718 | src_faults = group_faults(p, src_nid); |
| 6719 | dst_faults = group_faults(p, dst_nid); |
| 6720 | } else { |
| 6721 | src_faults = task_faults(p, src_nid); |
| 6722 | dst_faults = task_faults(p, dst_nid); |
| 6723 | } |
| 6724 | |
| 6725 | return dst_faults < src_faults; |
Mel Gorman | 7a0f308 | 2013-10-07 11:29:01 +0100 | [diff] [blame] | 6726 | } |
| 6727 | |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6728 | #else |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6729 | static inline int migrate_degrades_locality(struct task_struct *p, |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6730 | struct lb_env *env) |
| 6731 | { |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6732 | return -1; |
Mel Gorman | 7a0f308 | 2013-10-07 11:29:01 +0100 | [diff] [blame] | 6733 | } |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6734 | #endif |
| 6735 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 6736 | /* |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6737 | * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? |
| 6738 | */ |
| 6739 | static |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 6740 | int can_migrate_task(struct task_struct *p, struct lb_env *env) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6741 | { |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6742 | int tsk_cache_hot; |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6743 | |
| 6744 | lockdep_assert_held(&env->src_rq->lock); |
| 6745 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6746 | /* |
| 6747 | * We do not migrate tasks that are: |
Joonsoo Kim | d319808 | 2013-04-23 17:27:40 +0900 | [diff] [blame] | 6748 | * 1) throttled_lb_pair, or |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6749 | * 2) cannot be migrated to this CPU due to cpus_allowed, or |
Joonsoo Kim | d319808 | 2013-04-23 17:27:40 +0900 | [diff] [blame] | 6750 | * 3) running (obviously), or |
| 6751 | * 4) are cache-hot on their current CPU. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6752 | */ |
Joonsoo Kim | d319808 | 2013-04-23 17:27:40 +0900 | [diff] [blame] | 6753 | if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) |
| 6754 | return 0; |
| 6755 | |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 6756 | if (!cpumask_test_cpu(env->dst_cpu, &p->cpus_allowed)) { |
Joonsoo Kim | e02e60c | 2013-04-23 17:27:42 +0900 | [diff] [blame] | 6757 | int cpu; |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6758 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 6759 | schedstat_inc(p->se.statistics.nr_failed_migrations_affine); |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6760 | |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 6761 | env->flags |= LBF_SOME_PINNED; |
| 6762 | |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6763 | /* |
| 6764 | * Remember if this task can be migrated to any other cpu in |
| 6765 | * our sched_group. We may want to revisit it if we couldn't |
| 6766 | * meet load balance goals by pulling other tasks on src_cpu. |
| 6767 | * |
Jeffrey Hugo | 65a4433 | 2017-06-07 13:18:57 -0600 | [diff] [blame] | 6768 | * Avoid computing new_dst_cpu for NEWLY_IDLE or if we have |
| 6769 | * already computed one in current iteration. |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6770 | */ |
Jeffrey Hugo | 65a4433 | 2017-06-07 13:18:57 -0600 | [diff] [blame] | 6771 | if (env->idle == CPU_NEWLY_IDLE || (env->flags & LBF_DST_PINNED)) |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6772 | return 0; |
| 6773 | |
Joonsoo Kim | e02e60c | 2013-04-23 17:27:42 +0900 | [diff] [blame] | 6774 | /* Prevent to re-select dst_cpu via env's cpus */ |
| 6775 | for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 6776 | if (cpumask_test_cpu(cpu, &p->cpus_allowed)) { |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 6777 | env->flags |= LBF_DST_PINNED; |
Joonsoo Kim | e02e60c | 2013-04-23 17:27:42 +0900 | [diff] [blame] | 6778 | env->new_dst_cpu = cpu; |
| 6779 | break; |
| 6780 | } |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6781 | } |
Joonsoo Kim | e02e60c | 2013-04-23 17:27:42 +0900 | [diff] [blame] | 6782 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6783 | return 0; |
| 6784 | } |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 6785 | |
| 6786 | /* Record that we found atleast one task that could run on dst_cpu */ |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 6787 | env->flags &= ~LBF_ALL_PINNED; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6788 | |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 6789 | if (task_running(env->src_rq, p)) { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 6790 | schedstat_inc(p->se.statistics.nr_failed_migrations_running); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6791 | return 0; |
| 6792 | } |
| 6793 | |
| 6794 | /* |
| 6795 | * Aggressive migration if: |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6796 | * 1) destination numa is preferred |
| 6797 | * 2) task is cache cold, or |
| 6798 | * 3) too many balance attempts have failed. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6799 | */ |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6800 | tsk_cache_hot = migrate_degrades_locality(p, env); |
| 6801 | if (tsk_cache_hot == -1) |
| 6802 | tsk_cache_hot = task_hot(p, env); |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6803 | |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6804 | if (tsk_cache_hot <= 0 || |
Kirill Tkhai | 7a96c23 | 2014-09-22 22:36:12 +0400 | [diff] [blame] | 6805 | env->sd->nr_balance_failed > env->sd->cache_nice_tries) { |
Srikar Dronamraju | 2a1ed24 | 2015-06-16 17:25:59 +0530 | [diff] [blame] | 6806 | if (tsk_cache_hot == 1) { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 6807 | schedstat_inc(env->sd->lb_hot_gained[env->idle]); |
| 6808 | schedstat_inc(p->se.statistics.nr_forced_migrations); |
Mel Gorman | 3a7053b | 2013-10-07 11:29:00 +0100 | [diff] [blame] | 6809 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6810 | return 1; |
| 6811 | } |
| 6812 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 6813 | schedstat_inc(p->se.statistics.nr_failed_migrations_hot); |
Zhang Hang | 4e2dcb7 | 2013-04-10 14:04:55 +0800 | [diff] [blame] | 6814 | return 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6815 | } |
| 6816 | |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6817 | /* |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6818 | * detach_task() -- detach the task for the migration specified in env |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6819 | */ |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6820 | static void detach_task(struct task_struct *p, struct lb_env *env) |
| 6821 | { |
| 6822 | lockdep_assert_held(&env->src_rq->lock); |
| 6823 | |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6824 | p->on_rq = TASK_ON_RQ_MIGRATING; |
Peter Zijlstra | 5704ac0 | 2017-02-21 17:15:21 +0100 | [diff] [blame] | 6825 | deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6826 | set_task_cpu(p, env->dst_cpu); |
| 6827 | } |
| 6828 | |
| 6829 | /* |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6830 | * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6831 | * part of active balancing operations within "domain". |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6832 | * |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6833 | * Returns a task if successful and NULL otherwise. |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6834 | */ |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6835 | static struct task_struct *detach_one_task(struct lb_env *env) |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6836 | { |
| 6837 | struct task_struct *p, *n; |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6838 | |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6839 | lockdep_assert_held(&env->src_rq->lock); |
| 6840 | |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6841 | list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) { |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6842 | if (!can_migrate_task(p, env)) |
| 6843 | continue; |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6844 | |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6845 | detach_task(p, env); |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6846 | |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6847 | /* |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6848 | * Right now, this is only the second place where |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6849 | * lb_gained[env->idle] is updated (other is detach_tasks) |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6850 | * so we can safely collect stats here rather than |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6851 | * inside detach_tasks(). |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6852 | */ |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 6853 | schedstat_inc(env->sd->lb_gained[env->idle]); |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6854 | return p; |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6855 | } |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 6856 | return NULL; |
Peter Zijlstra | 897c395 | 2009-12-17 17:45:42 +0100 | [diff] [blame] | 6857 | } |
| 6858 | |
Peter Zijlstra | eb95308 | 2012-04-17 13:38:40 +0200 | [diff] [blame] | 6859 | static const unsigned int sched_nr_migrate_break = 32; |
| 6860 | |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6861 | /* |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6862 | * detach_tasks() -- tries to detach up to imbalance weighted load from |
| 6863 | * busiest_rq, as part of a balancing operation within domain "sd". |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6864 | * |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6865 | * Returns number of detached tasks if successful and 0 otherwise. |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6866 | */ |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6867 | static int detach_tasks(struct lb_env *env) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6868 | { |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6869 | struct list_head *tasks = &env->src_rq->cfs_tasks; |
| 6870 | struct task_struct *p; |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6871 | unsigned long load; |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6872 | int detached = 0; |
| 6873 | |
| 6874 | lockdep_assert_held(&env->src_rq->lock); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6875 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 6876 | if (env->imbalance <= 0) |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6877 | return 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6878 | |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6879 | while (!list_empty(tasks)) { |
Yuyang Du | 985d3a4 | 2015-07-06 06:11:51 +0800 | [diff] [blame] | 6880 | /* |
| 6881 | * We don't want to steal all, otherwise we may be treated likewise, |
| 6882 | * which could at worst lead to a livelock crash. |
| 6883 | */ |
| 6884 | if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) |
| 6885 | break; |
| 6886 | |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6887 | p = list_first_entry(tasks, struct task_struct, se.group_node); |
| 6888 | |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6889 | env->loop++; |
| 6890 | /* We've more or less seen every task there is, call it quits */ |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6891 | if (env->loop > env->loop_max) |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6892 | break; |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6893 | |
| 6894 | /* take a breather every nr_migrate tasks */ |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6895 | if (env->loop > env->loop_break) { |
Peter Zijlstra | eb95308 | 2012-04-17 13:38:40 +0200 | [diff] [blame] | 6896 | env->loop_break += sched_nr_migrate_break; |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 6897 | env->flags |= LBF_NEED_BREAK; |
Peter Zijlstra | ee00e66 | 2009-12-17 17:25:20 +0100 | [diff] [blame] | 6898 | break; |
Peter Zijlstra | a195f00 | 2011-09-22 15:30:18 +0200 | [diff] [blame] | 6899 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6900 | |
Joonsoo Kim | d319808 | 2013-04-23 17:27:40 +0900 | [diff] [blame] | 6901 | if (!can_migrate_task(p, env)) |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6902 | goto next; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6903 | |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6904 | load = task_h_load(p); |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6905 | |
Peter Zijlstra | eb95308 | 2012-04-17 13:38:40 +0200 | [diff] [blame] | 6906 | if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed) |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6907 | goto next; |
| 6908 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 6909 | if ((load / 2) > env->imbalance) |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6910 | goto next; |
| 6911 | |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6912 | detach_task(p, env); |
| 6913 | list_add(&p->se.group_node, &env->tasks); |
| 6914 | |
| 6915 | detached++; |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 6916 | env->imbalance -= load; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6917 | |
| 6918 | #ifdef CONFIG_PREEMPT |
Peter Zijlstra | ee00e66 | 2009-12-17 17:25:20 +0100 | [diff] [blame] | 6919 | /* |
| 6920 | * NEWIDLE balancing is a source of latency, so preemptible |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6921 | * kernels will stop after the first task is detached to minimize |
Peter Zijlstra | ee00e66 | 2009-12-17 17:25:20 +0100 | [diff] [blame] | 6922 | * the critical section. |
| 6923 | */ |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6924 | if (env->idle == CPU_NEWLY_IDLE) |
Peter Zijlstra | ee00e66 | 2009-12-17 17:25:20 +0100 | [diff] [blame] | 6925 | break; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6926 | #endif |
| 6927 | |
Peter Zijlstra | ee00e66 | 2009-12-17 17:25:20 +0100 | [diff] [blame] | 6928 | /* |
| 6929 | * We only want to steal up to the prescribed amount of |
| 6930 | * weighted load. |
| 6931 | */ |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 6932 | if (env->imbalance <= 0) |
Peter Zijlstra | ee00e66 | 2009-12-17 17:25:20 +0100 | [diff] [blame] | 6933 | break; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6934 | |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 6935 | continue; |
| 6936 | next: |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6937 | list_move_tail(&p->se.group_node, tasks); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6938 | } |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 6939 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6940 | /* |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6941 | * Right now, this is one of only two places we collect this stat |
| 6942 | * so we can safely collect detach_one_task() stats here rather |
| 6943 | * than inside detach_one_task(). |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6944 | */ |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 6945 | schedstat_add(env->sd->lb_gained[env->idle], detached); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6946 | |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6947 | return detached; |
| 6948 | } |
| 6949 | |
| 6950 | /* |
| 6951 | * attach_task() -- attach the task detached by detach_task() to its new rq. |
| 6952 | */ |
| 6953 | static void attach_task(struct rq *rq, struct task_struct *p) |
| 6954 | { |
| 6955 | lockdep_assert_held(&rq->lock); |
| 6956 | |
| 6957 | BUG_ON(task_rq(p) != rq); |
Peter Zijlstra | 5704ac0 | 2017-02-21 17:15:21 +0100 | [diff] [blame] | 6958 | activate_task(rq, p, ENQUEUE_NOCLOCK); |
Joonwoo Park | 3ea94de | 2015-11-12 19:38:54 -0800 | [diff] [blame] | 6959 | p->on_rq = TASK_ON_RQ_QUEUED; |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6960 | check_preempt_curr(rq, p, 0); |
| 6961 | } |
| 6962 | |
| 6963 | /* |
| 6964 | * attach_one_task() -- attaches the task returned from detach_one_task() to |
| 6965 | * its new rq. |
| 6966 | */ |
| 6967 | static void attach_one_task(struct rq *rq, struct task_struct *p) |
| 6968 | { |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 6969 | struct rq_flags rf; |
| 6970 | |
| 6971 | rq_lock(rq, &rf); |
Peter Zijlstra | 5704ac0 | 2017-02-21 17:15:21 +0100 | [diff] [blame] | 6972 | update_rq_clock(rq); |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6973 | attach_task(rq, p); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 6974 | rq_unlock(rq, &rf); |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6975 | } |
| 6976 | |
| 6977 | /* |
| 6978 | * attach_tasks() -- attaches all tasks detached by detach_tasks() to their |
| 6979 | * new rq. |
| 6980 | */ |
| 6981 | static void attach_tasks(struct lb_env *env) |
| 6982 | { |
| 6983 | struct list_head *tasks = &env->tasks; |
| 6984 | struct task_struct *p; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 6985 | struct rq_flags rf; |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6986 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 6987 | rq_lock(env->dst_rq, &rf); |
Peter Zijlstra | 5704ac0 | 2017-02-21 17:15:21 +0100 | [diff] [blame] | 6988 | update_rq_clock(env->dst_rq); |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 6989 | |
| 6990 | while (!list_empty(tasks)) { |
| 6991 | p = list_first_entry(tasks, struct task_struct, se.group_node); |
| 6992 | list_del_init(&p->se.group_node); |
| 6993 | |
| 6994 | attach_task(env->dst_rq, p); |
| 6995 | } |
| 6996 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 6997 | rq_unlock(env->dst_rq, &rf); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 6998 | } |
| 6999 | |
Peter Zijlstra | 230059de | 2009-12-17 17:47:12 +0100 | [diff] [blame] | 7000 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 7001 | |
| 7002 | static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) |
| 7003 | { |
| 7004 | if (cfs_rq->load.weight) |
| 7005 | return false; |
| 7006 | |
| 7007 | if (cfs_rq->avg.load_sum) |
| 7008 | return false; |
| 7009 | |
| 7010 | if (cfs_rq->avg.util_sum) |
| 7011 | return false; |
| 7012 | |
| 7013 | if (cfs_rq->runnable_load_sum) |
| 7014 | return false; |
| 7015 | |
| 7016 | return true; |
| 7017 | } |
| 7018 | |
Paul Turner | 48a1675 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 7019 | static void update_blocked_averages(int cpu) |
Peter Zijlstra | 9e3081c | 2010-11-15 15:47:02 -0800 | [diff] [blame] | 7020 | { |
Peter Zijlstra | 9e3081c | 2010-11-15 15:47:02 -0800 | [diff] [blame] | 7021 | struct rq *rq = cpu_rq(cpu); |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 7022 | struct cfs_rq *cfs_rq, *pos; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 7023 | struct rq_flags rf; |
Peter Zijlstra | 9e3081c | 2010-11-15 15:47:02 -0800 | [diff] [blame] | 7024 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 7025 | rq_lock_irqsave(rq, &rf); |
Paul Turner | 48a1675 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 7026 | update_rq_clock(rq); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 7027 | |
Peter Zijlstra | 9763b67 | 2011-07-13 13:09:25 +0200 | [diff] [blame] | 7028 | /* |
| 7029 | * Iterates the task_group tree in a bottom up fashion, see |
| 7030 | * list_add_leaf_cfs_rq() for details. |
| 7031 | */ |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 7032 | for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { |
Vincent Guittot | bc42789 | 2017-03-17 14:47:22 +0100 | [diff] [blame] | 7033 | struct sched_entity *se; |
| 7034 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 7035 | /* throttled entities do not contribute to load */ |
| 7036 | if (throttled_hierarchy(cfs_rq)) |
| 7037 | continue; |
Paul Turner | 48a1675 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 7038 | |
Viresh Kumar | 3a123bb | 2017-05-24 10:59:56 +0530 | [diff] [blame] | 7039 | if (update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq)) |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 7040 | update_tg_load_avg(cfs_rq, 0); |
Vincent Guittot | 4e51607 | 2016-11-08 10:53:46 +0100 | [diff] [blame] | 7041 | |
Vincent Guittot | bc42789 | 2017-03-17 14:47:22 +0100 | [diff] [blame] | 7042 | /* Propagate pending load changes to the parent, if any: */ |
| 7043 | se = cfs_rq->tg->se[cpu]; |
| 7044 | if (se && !skip_blocked_update(se)) |
| 7045 | update_load_avg(se, 0); |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 7046 | |
| 7047 | /* |
| 7048 | * There can be a lot of idle CPU cgroups. Don't let fully |
| 7049 | * decayed cfs_rqs linger on the list. |
| 7050 | */ |
| 7051 | if (cfs_rq_is_decayed(cfs_rq)) |
| 7052 | list_del_leaf_cfs_rq(cfs_rq); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 7053 | } |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 7054 | rq_unlock_irqrestore(rq, &rf); |
Peter Zijlstra | 9e3081c | 2010-11-15 15:47:02 -0800 | [diff] [blame] | 7055 | } |
| 7056 | |
Peter Zijlstra | 9763b67 | 2011-07-13 13:09:25 +0200 | [diff] [blame] | 7057 | /* |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7058 | * Compute the hierarchical load factor for cfs_rq and all its ascendants. |
Peter Zijlstra | 9763b67 | 2011-07-13 13:09:25 +0200 | [diff] [blame] | 7059 | * This needs to be done in a top-down fashion because the load of a child |
| 7060 | * group is a fraction of its parents load. |
| 7061 | */ |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7062 | static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) |
Peter Zijlstra | 9763b67 | 2011-07-13 13:09:25 +0200 | [diff] [blame] | 7063 | { |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7064 | struct rq *rq = rq_of(cfs_rq); |
| 7065 | struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; |
Peter Zijlstra | a35b646 | 2012-08-08 21:46:40 +0200 | [diff] [blame] | 7066 | unsigned long now = jiffies; |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7067 | unsigned long load; |
Peter Zijlstra | a35b646 | 2012-08-08 21:46:40 +0200 | [diff] [blame] | 7068 | |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7069 | if (cfs_rq->last_h_load_update == now) |
Peter Zijlstra | a35b646 | 2012-08-08 21:46:40 +0200 | [diff] [blame] | 7070 | return; |
| 7071 | |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7072 | cfs_rq->h_load_next = NULL; |
| 7073 | for_each_sched_entity(se) { |
| 7074 | cfs_rq = cfs_rq_of(se); |
| 7075 | cfs_rq->h_load_next = se; |
| 7076 | if (cfs_rq->last_h_load_update == now) |
| 7077 | break; |
| 7078 | } |
Peter Zijlstra | a35b646 | 2012-08-08 21:46:40 +0200 | [diff] [blame] | 7079 | |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7080 | if (!se) { |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 7081 | cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7082 | cfs_rq->last_h_load_update = now; |
| 7083 | } |
| 7084 | |
| 7085 | while ((se = cfs_rq->h_load_next) != NULL) { |
| 7086 | load = cfs_rq->h_load; |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 7087 | load = div64_ul(load * se->avg.load_avg, |
| 7088 | cfs_rq_load_avg(cfs_rq) + 1); |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7089 | cfs_rq = group_cfs_rq(se); |
| 7090 | cfs_rq->h_load = load; |
| 7091 | cfs_rq->last_h_load_update = now; |
| 7092 | } |
Peter Zijlstra | 9763b67 | 2011-07-13 13:09:25 +0200 | [diff] [blame] | 7093 | } |
| 7094 | |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 7095 | static unsigned long task_h_load(struct task_struct *p) |
Peter Zijlstra | 230059de | 2009-12-17 17:47:12 +0100 | [diff] [blame] | 7096 | { |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 7097 | struct cfs_rq *cfs_rq = task_cfs_rq(p); |
Peter Zijlstra | 230059de | 2009-12-17 17:47:12 +0100 | [diff] [blame] | 7098 | |
Vladimir Davydov | 6852079 | 2013-07-15 17:49:19 +0400 | [diff] [blame] | 7099 | update_cfs_rq_h_load(cfs_rq); |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 7100 | return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, |
Yuyang Du | 7ea241a | 2015-07-15 08:04:42 +0800 | [diff] [blame] | 7101 | cfs_rq_load_avg(cfs_rq) + 1); |
Peter Zijlstra | 230059de | 2009-12-17 17:47:12 +0100 | [diff] [blame] | 7102 | } |
| 7103 | #else |
Paul Turner | 48a1675 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 7104 | static inline void update_blocked_averages(int cpu) |
Peter Zijlstra | 9e3081c | 2010-11-15 15:47:02 -0800 | [diff] [blame] | 7105 | { |
Vincent Guittot | 6c1d47c | 2015-07-15 08:04:38 +0800 | [diff] [blame] | 7106 | struct rq *rq = cpu_rq(cpu); |
| 7107 | struct cfs_rq *cfs_rq = &rq->cfs; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 7108 | struct rq_flags rf; |
Vincent Guittot | 6c1d47c | 2015-07-15 08:04:38 +0800 | [diff] [blame] | 7109 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 7110 | rq_lock_irqsave(rq, &rf); |
Vincent Guittot | 6c1d47c | 2015-07-15 08:04:38 +0800 | [diff] [blame] | 7111 | update_rq_clock(rq); |
Viresh Kumar | 3a123bb | 2017-05-24 10:59:56 +0530 | [diff] [blame] | 7112 | update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 7113 | rq_unlock_irqrestore(rq, &rf); |
Peter Zijlstra | 9e3081c | 2010-11-15 15:47:02 -0800 | [diff] [blame] | 7114 | } |
| 7115 | |
Peter Zijlstra | 367456c | 2012-02-20 21:49:09 +0100 | [diff] [blame] | 7116 | static unsigned long task_h_load(struct task_struct *p) |
| 7117 | { |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 7118 | return p->se.avg.load_avg; |
Peter Zijlstra | 230059de | 2009-12-17 17:47:12 +0100 | [diff] [blame] | 7119 | } |
| 7120 | #endif |
| 7121 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7122 | /********** Helpers for find_busiest_group ************************/ |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7123 | |
| 7124 | enum group_type { |
| 7125 | group_other = 0, |
| 7126 | group_imbalanced, |
| 7127 | group_overloaded, |
| 7128 | }; |
| 7129 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7130 | /* |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7131 | * sg_lb_stats - stats of a sched_group required for load_balancing |
| 7132 | */ |
| 7133 | struct sg_lb_stats { |
| 7134 | unsigned long avg_load; /*Avg load across the CPUs of the group */ |
| 7135 | unsigned long group_load; /* Total load over the CPUs of the group */ |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7136 | unsigned long sum_weighted_load; /* Weighted load of group's tasks */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7137 | unsigned long load_per_task; |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7138 | unsigned long group_capacity; |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 7139 | unsigned long group_util; /* Total utilization of the group */ |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7140 | unsigned int sum_nr_running; /* Nr tasks running in the group */ |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7141 | unsigned int idle_cpus; |
| 7142 | unsigned int group_weight; |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7143 | enum group_type group_type; |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7144 | int group_no_capacity; |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 7145 | #ifdef CONFIG_NUMA_BALANCING |
| 7146 | unsigned int nr_numa_running; |
| 7147 | unsigned int nr_preferred_running; |
| 7148 | #endif |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7149 | }; |
| 7150 | |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7151 | /* |
| 7152 | * sd_lb_stats - Structure to store the statistics of a sched_domain |
| 7153 | * during load balancing. |
| 7154 | */ |
| 7155 | struct sd_lb_stats { |
| 7156 | struct sched_group *busiest; /* Busiest group in this sd */ |
| 7157 | struct sched_group *local; /* Local group in this sd */ |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 7158 | unsigned long total_running; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7159 | unsigned long total_load; /* Total load of all groups in sd */ |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7160 | unsigned long total_capacity; /* Total capacity of all groups in sd */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7161 | unsigned long avg_load; /* Average load across all groups in sd */ |
| 7162 | |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7163 | struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7164 | struct sg_lb_stats local_stat; /* Statistics of the local group */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7165 | }; |
| 7166 | |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7167 | static inline void init_sd_lb_stats(struct sd_lb_stats *sds) |
| 7168 | { |
| 7169 | /* |
| 7170 | * Skimp on the clearing to avoid duplicate work. We can avoid clearing |
| 7171 | * local_stat because update_sg_lb_stats() does a full clear/assignment. |
| 7172 | * We must however clear busiest_stat::avg_load because |
| 7173 | * update_sd_pick_busiest() reads this before assignment. |
| 7174 | */ |
| 7175 | *sds = (struct sd_lb_stats){ |
| 7176 | .busiest = NULL, |
| 7177 | .local = NULL, |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 7178 | .total_running = 0UL, |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7179 | .total_load = 0UL, |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7180 | .total_capacity = 0UL, |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7181 | .busiest_stat = { |
| 7182 | .avg_load = 0UL, |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7183 | .sum_nr_running = 0, |
| 7184 | .group_type = group_other, |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7185 | }, |
| 7186 | }; |
| 7187 | } |
| 7188 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7189 | /** |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7190 | * get_sd_load_idx - Obtain the load index for a given sched domain. |
| 7191 | * @sd: The sched_domain whose load_idx is to be obtained. |
Kamalesh Babulal | ed1b773 | 2013-10-13 23:06:15 +0530 | [diff] [blame] | 7192 | * @idle: The idle status of the CPU for whose sd load_idx is obtained. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7193 | * |
| 7194 | * Return: The load index. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7195 | */ |
| 7196 | static inline int get_sd_load_idx(struct sched_domain *sd, |
| 7197 | enum cpu_idle_type idle) |
| 7198 | { |
| 7199 | int load_idx; |
| 7200 | |
| 7201 | switch (idle) { |
| 7202 | case CPU_NOT_IDLE: |
| 7203 | load_idx = sd->busy_idx; |
| 7204 | break; |
| 7205 | |
| 7206 | case CPU_NEWLY_IDLE: |
| 7207 | load_idx = sd->newidle_idx; |
| 7208 | break; |
| 7209 | default: |
| 7210 | load_idx = sd->idle_idx; |
| 7211 | break; |
| 7212 | } |
| 7213 | |
| 7214 | return load_idx; |
| 7215 | } |
| 7216 | |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7217 | static unsigned long scale_rt_capacity(int cpu) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7218 | { |
| 7219 | struct rq *rq = cpu_rq(cpu); |
Vincent Guittot | b5b4860 | 2015-02-27 16:54:08 +0100 | [diff] [blame] | 7220 | u64 total, used, age_stamp, avg; |
Peter Zijlstra | cadefd3 | 2014-02-27 10:40:35 +0100 | [diff] [blame] | 7221 | s64 delta; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7222 | |
Peter Zijlstra | b654f7d | 2012-05-22 14:04:28 +0200 | [diff] [blame] | 7223 | /* |
| 7224 | * Since we're reading these variables without serialization make sure |
| 7225 | * we read them once before doing sanity checks on them. |
| 7226 | */ |
Jason Low | 316c1608d | 2015-04-28 13:00:20 -0700 | [diff] [blame] | 7227 | age_stamp = READ_ONCE(rq->age_stamp); |
| 7228 | avg = READ_ONCE(rq->rt_avg); |
Peter Zijlstra | cebde6d | 2015-01-05 11:18:10 +0100 | [diff] [blame] | 7229 | delta = __rq_clock_broken(rq) - age_stamp; |
Venkatesh Pallipadi | aa48380 | 2010-10-04 17:03:22 -0700 | [diff] [blame] | 7230 | |
Peter Zijlstra | cadefd3 | 2014-02-27 10:40:35 +0100 | [diff] [blame] | 7231 | if (unlikely(delta < 0)) |
| 7232 | delta = 0; |
| 7233 | |
| 7234 | total = sched_avg_period() + delta; |
Peter Zijlstra | b654f7d | 2012-05-22 14:04:28 +0200 | [diff] [blame] | 7235 | |
Vincent Guittot | b5b4860 | 2015-02-27 16:54:08 +0100 | [diff] [blame] | 7236 | used = div_u64(avg, total); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7237 | |
Vincent Guittot | b5b4860 | 2015-02-27 16:54:08 +0100 | [diff] [blame] | 7238 | if (likely(used < SCHED_CAPACITY_SCALE)) |
| 7239 | return SCHED_CAPACITY_SCALE - used; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7240 | |
Vincent Guittot | b5b4860 | 2015-02-27 16:54:08 +0100 | [diff] [blame] | 7241 | return 1; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7242 | } |
| 7243 | |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7244 | static void update_cpu_capacity(struct sched_domain *sd, int cpu) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7245 | { |
Morten Rasmussen | 8cd5601 | 2015-08-14 17:23:10 +0100 | [diff] [blame] | 7246 | unsigned long capacity = arch_scale_cpu_capacity(sd, cpu); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7247 | struct sched_group *sdg = sd->groups; |
| 7248 | |
Vincent Guittot | ca6d75e | 2015-02-27 16:54:09 +0100 | [diff] [blame] | 7249 | cpu_rq(cpu)->cpu_capacity_orig = capacity; |
Srivatsa Vaddagiri | 9d5efe0 | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7250 | |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7251 | capacity *= scale_rt_capacity(cpu); |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7252 | capacity >>= SCHED_CAPACITY_SHIFT; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7253 | |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7254 | if (!capacity) |
| 7255 | capacity = 1; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7256 | |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7257 | cpu_rq(cpu)->cpu_capacity = capacity; |
| 7258 | sdg->sgc->capacity = capacity; |
Morten Rasmussen | bf475ce | 2016-10-14 14:41:09 +0100 | [diff] [blame] | 7259 | sdg->sgc->min_capacity = capacity; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7260 | } |
| 7261 | |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7262 | void update_group_capacity(struct sched_domain *sd, int cpu) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7263 | { |
| 7264 | struct sched_domain *child = sd->child; |
| 7265 | struct sched_group *group, *sdg = sd->groups; |
Morten Rasmussen | bf475ce | 2016-10-14 14:41:09 +0100 | [diff] [blame] | 7266 | unsigned long capacity, min_capacity; |
Vincent Guittot | 4ec4412 | 2011-12-12 20:21:08 +0100 | [diff] [blame] | 7267 | unsigned long interval; |
| 7268 | |
| 7269 | interval = msecs_to_jiffies(sd->balance_interval); |
| 7270 | interval = clamp(interval, 1UL, max_load_balance_interval); |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7271 | sdg->sgc->next_update = jiffies + interval; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7272 | |
| 7273 | if (!child) { |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7274 | update_cpu_capacity(sd, cpu); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7275 | return; |
| 7276 | } |
| 7277 | |
Vincent Guittot | dc7ff76 | 2015-03-03 11:35:03 +0100 | [diff] [blame] | 7278 | capacity = 0; |
Morten Rasmussen | bf475ce | 2016-10-14 14:41:09 +0100 | [diff] [blame] | 7279 | min_capacity = ULONG_MAX; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7280 | |
Peter Zijlstra | 74a5ce2 | 2012-05-23 18:00:43 +0200 | [diff] [blame] | 7281 | if (child->flags & SD_OVERLAP) { |
| 7282 | /* |
| 7283 | * SD_OVERLAP domains cannot assume that child groups |
| 7284 | * span the current group. |
| 7285 | */ |
| 7286 | |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 7287 | for_each_cpu(cpu, sched_group_span(sdg)) { |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7288 | struct sched_group_capacity *sgc; |
Srikar Dronamraju | 9abf24d | 2013-11-12 22:11:26 +0530 | [diff] [blame] | 7289 | struct rq *rq = cpu_rq(cpu); |
Peter Zijlstra | 863bffc | 2013-08-28 11:44:39 +0200 | [diff] [blame] | 7290 | |
Srikar Dronamraju | 9abf24d | 2013-11-12 22:11:26 +0530 | [diff] [blame] | 7291 | /* |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7292 | * build_sched_domains() -> init_sched_groups_capacity() |
Srikar Dronamraju | 9abf24d | 2013-11-12 22:11:26 +0530 | [diff] [blame] | 7293 | * gets here before we've attached the domains to the |
| 7294 | * runqueues. |
| 7295 | * |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7296 | * Use capacity_of(), which is set irrespective of domains |
| 7297 | * in update_cpu_capacity(). |
Srikar Dronamraju | 9abf24d | 2013-11-12 22:11:26 +0530 | [diff] [blame] | 7298 | * |
Vincent Guittot | dc7ff76 | 2015-03-03 11:35:03 +0100 | [diff] [blame] | 7299 | * This avoids capacity from being 0 and |
Srikar Dronamraju | 9abf24d | 2013-11-12 22:11:26 +0530 | [diff] [blame] | 7300 | * causing divide-by-zero issues on boot. |
Srikar Dronamraju | 9abf24d | 2013-11-12 22:11:26 +0530 | [diff] [blame] | 7301 | */ |
| 7302 | if (unlikely(!rq->sd)) { |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7303 | capacity += capacity_of(cpu); |
Morten Rasmussen | bf475ce | 2016-10-14 14:41:09 +0100 | [diff] [blame] | 7304 | } else { |
| 7305 | sgc = rq->sd->groups->sgc; |
| 7306 | capacity += sgc->capacity; |
Srikar Dronamraju | 9abf24d | 2013-11-12 22:11:26 +0530 | [diff] [blame] | 7307 | } |
| 7308 | |
Morten Rasmussen | bf475ce | 2016-10-14 14:41:09 +0100 | [diff] [blame] | 7309 | min_capacity = min(capacity, min_capacity); |
Peter Zijlstra | 863bffc | 2013-08-28 11:44:39 +0200 | [diff] [blame] | 7310 | } |
Peter Zijlstra | 74a5ce2 | 2012-05-23 18:00:43 +0200 | [diff] [blame] | 7311 | } else { |
| 7312 | /* |
| 7313 | * !SD_OVERLAP domains can assume that child groups |
| 7314 | * span the current group. |
Byungchul Park | 97a7142 | 2015-07-05 18:33:48 +0900 | [diff] [blame] | 7315 | */ |
Peter Zijlstra | 74a5ce2 | 2012-05-23 18:00:43 +0200 | [diff] [blame] | 7316 | |
| 7317 | group = child->groups; |
| 7318 | do { |
Morten Rasmussen | bf475ce | 2016-10-14 14:41:09 +0100 | [diff] [blame] | 7319 | struct sched_group_capacity *sgc = group->sgc; |
| 7320 | |
| 7321 | capacity += sgc->capacity; |
| 7322 | min_capacity = min(sgc->min_capacity, min_capacity); |
Peter Zijlstra | 74a5ce2 | 2012-05-23 18:00:43 +0200 | [diff] [blame] | 7323 | group = group->next; |
| 7324 | } while (group != child->groups); |
| 7325 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7326 | |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7327 | sdg->sgc->capacity = capacity; |
Morten Rasmussen | bf475ce | 2016-10-14 14:41:09 +0100 | [diff] [blame] | 7328 | sdg->sgc->min_capacity = min_capacity; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7329 | } |
| 7330 | |
Srivatsa Vaddagiri | 9d5efe0 | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7331 | /* |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7332 | * Check whether the capacity of the rq has been noticeably reduced by side |
| 7333 | * activity. The imbalance_pct is used for the threshold. |
| 7334 | * Return true is the capacity is reduced |
Srivatsa Vaddagiri | 9d5efe0 | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7335 | */ |
| 7336 | static inline int |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7337 | check_cpu_capacity(struct rq *rq, struct sched_domain *sd) |
Srivatsa Vaddagiri | 9d5efe0 | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7338 | { |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7339 | return ((rq->cpu_capacity * sd->imbalance_pct) < |
| 7340 | (rq->cpu_capacity_orig * 100)); |
Srivatsa Vaddagiri | 9d5efe0 | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7341 | } |
| 7342 | |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7343 | /* |
| 7344 | * Group imbalance indicates (and tries to solve) the problem where balancing |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 7345 | * groups is inadequate due to ->cpus_allowed constraints. |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7346 | * |
| 7347 | * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a |
| 7348 | * cpumask covering 1 cpu of the first group and 3 cpus of the second group. |
| 7349 | * Something like: |
| 7350 | * |
Ingo Molnar | 2b4d5b2 | 2016-11-23 07:37:00 +0100 | [diff] [blame] | 7351 | * { 0 1 2 3 } { 4 5 6 7 } |
| 7352 | * * * * * |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7353 | * |
| 7354 | * If we were to balance group-wise we'd place two tasks in the first group and |
| 7355 | * two tasks in the second group. Clearly this is undesired as it will overload |
| 7356 | * cpu 3 and leave one of the cpus in the second group unused. |
| 7357 | * |
| 7358 | * The current solution to this issue is detecting the skew in the first group |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 7359 | * by noticing the lower domain failed to reach balance and had difficulty |
| 7360 | * moving tasks due to affinity constraints. |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7361 | * |
| 7362 | * When this is so detected; this group becomes a candidate for busiest; see |
Kamalesh Babulal | ed1b773 | 2013-10-13 23:06:15 +0530 | [diff] [blame] | 7363 | * update_sd_pick_busiest(). And calculate_imbalance() and |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 7364 | * find_busiest_group() avoid some of the usual balance conditions to allow it |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7365 | * to create an effective group imbalance. |
| 7366 | * |
| 7367 | * This is a somewhat tricky proposition since the next run might not find the |
| 7368 | * group imbalance and decide the groups need to be balanced again. A most |
| 7369 | * subtle and fragile situation. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7370 | */ |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7371 | |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 7372 | static inline int sg_imbalanced(struct sched_group *group) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7373 | { |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7374 | return group->sgc->imbalance; |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7375 | } |
| 7376 | |
Peter Zijlstra | b37d931 | 2013-08-28 11:50:34 +0200 | [diff] [blame] | 7377 | /* |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7378 | * group_has_capacity returns true if the group has spare capacity that could |
| 7379 | * be used by some tasks. |
| 7380 | * We consider that a group has spare capacity if the * number of task is |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 7381 | * smaller than the number of CPUs or if the utilization is lower than the |
| 7382 | * available capacity for CFS tasks. |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7383 | * For the latter, we use a threshold to stabilize the state, to take into |
| 7384 | * account the variance of the tasks' load and to return true if the available |
| 7385 | * capacity in meaningful for the load balancer. |
| 7386 | * As an example, an available capacity of 1% can appear but it doesn't make |
| 7387 | * any benefit for the load balance. |
Peter Zijlstra | b37d931 | 2013-08-28 11:50:34 +0200 | [diff] [blame] | 7388 | */ |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7389 | static inline bool |
| 7390 | group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs) |
Peter Zijlstra | b37d931 | 2013-08-28 11:50:34 +0200 | [diff] [blame] | 7391 | { |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7392 | if (sgs->sum_nr_running < sgs->group_weight) |
| 7393 | return true; |
Peter Zijlstra | b37d931 | 2013-08-28 11:50:34 +0200 | [diff] [blame] | 7394 | |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7395 | if ((sgs->group_capacity * 100) > |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 7396 | (sgs->group_util * env->sd->imbalance_pct)) |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7397 | return true; |
Peter Zijlstra | b37d931 | 2013-08-28 11:50:34 +0200 | [diff] [blame] | 7398 | |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7399 | return false; |
Peter Zijlstra | b37d931 | 2013-08-28 11:50:34 +0200 | [diff] [blame] | 7400 | } |
| 7401 | |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7402 | /* |
| 7403 | * group_is_overloaded returns true if the group has more tasks than it can |
| 7404 | * handle. |
| 7405 | * group_is_overloaded is not equals to !group_has_capacity because a group |
| 7406 | * with the exact right number of tasks, has no more spare capacity but is not |
| 7407 | * overloaded so both group_has_capacity and group_is_overloaded return |
| 7408 | * false. |
| 7409 | */ |
| 7410 | static inline bool |
| 7411 | group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs) |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7412 | { |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7413 | if (sgs->sum_nr_running <= sgs->group_weight) |
| 7414 | return false; |
| 7415 | |
| 7416 | if ((sgs->group_capacity * 100) < |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 7417 | (sgs->group_util * env->sd->imbalance_pct)) |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7418 | return true; |
| 7419 | |
| 7420 | return false; |
| 7421 | } |
| 7422 | |
Morten Rasmussen | 9e0994c | 2016-10-14 14:41:10 +0100 | [diff] [blame] | 7423 | /* |
| 7424 | * group_smaller_cpu_capacity: Returns true if sched_group sg has smaller |
| 7425 | * per-CPU capacity than sched_group ref. |
| 7426 | */ |
| 7427 | static inline bool |
| 7428 | group_smaller_cpu_capacity(struct sched_group *sg, struct sched_group *ref) |
| 7429 | { |
| 7430 | return sg->sgc->min_capacity * capacity_margin < |
| 7431 | ref->sgc->min_capacity * 1024; |
| 7432 | } |
| 7433 | |
Leo Yan | 79a89f9 | 2015-09-15 18:56:45 +0800 | [diff] [blame] | 7434 | static inline enum |
| 7435 | group_type group_classify(struct sched_group *group, |
| 7436 | struct sg_lb_stats *sgs) |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7437 | { |
| 7438 | if (sgs->group_no_capacity) |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7439 | return group_overloaded; |
| 7440 | |
| 7441 | if (sg_imbalanced(group)) |
| 7442 | return group_imbalanced; |
| 7443 | |
| 7444 | return group_other; |
| 7445 | } |
| 7446 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7447 | /** |
| 7448 | * update_sg_lb_stats - Update sched_group's statistics for load balancing. |
| 7449 | * @env: The load balancing environment. |
| 7450 | * @group: sched_group whose statistics are to be updated. |
| 7451 | * @load_idx: Load index of sched_domain of this_cpu for load calc. |
| 7452 | * @local_group: Does group contain this_cpu. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7453 | * @sgs: variable to hold the statistics for this group. |
Masanari Iida | cd3bd4e | 2014-07-28 12:38:06 +0900 | [diff] [blame] | 7454 | * @overload: Indicate more than one runnable task for any CPU. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7455 | */ |
| 7456 | static inline void update_sg_lb_stats(struct lb_env *env, |
| 7457 | struct sched_group *group, int load_idx, |
Tim Chen | 4486edd | 2014-06-23 12:16:49 -0700 | [diff] [blame] | 7458 | int local_group, struct sg_lb_stats *sgs, |
| 7459 | bool *overload) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7460 | { |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7461 | unsigned long load; |
Waiman Long | a426f99 | 2015-11-25 14:09:38 -0500 | [diff] [blame] | 7462 | int i, nr_running; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7463 | |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7464 | memset(sgs, 0, sizeof(*sgs)); |
| 7465 | |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 7466 | for_each_cpu_and(i, sched_group_span(group), env->cpus) { |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7467 | struct rq *rq = cpu_rq(i); |
| 7468 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7469 | /* Bias balancing toward cpus of our domain */ |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 7470 | if (local_group) |
Peter Zijlstra | 04f733b | 2012-05-11 00:12:02 +0200 | [diff] [blame] | 7471 | load = target_load(i, load_idx); |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 7472 | else |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7473 | load = source_load(i, load_idx); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7474 | |
| 7475 | sgs->group_load += load; |
Dietmar Eggemann | 9e91d61 | 2015-08-14 17:23:12 +0100 | [diff] [blame] | 7476 | sgs->group_util += cpu_util(i); |
Vincent Guittot | 65fdac0 | 2014-08-26 13:06:46 +0200 | [diff] [blame] | 7477 | sgs->sum_nr_running += rq->cfs.h_nr_running; |
Tim Chen | 4486edd | 2014-06-23 12:16:49 -0700 | [diff] [blame] | 7478 | |
Waiman Long | a426f99 | 2015-11-25 14:09:38 -0500 | [diff] [blame] | 7479 | nr_running = rq->nr_running; |
| 7480 | if (nr_running > 1) |
Tim Chen | 4486edd | 2014-06-23 12:16:49 -0700 | [diff] [blame] | 7481 | *overload = true; |
| 7482 | |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 7483 | #ifdef CONFIG_NUMA_BALANCING |
| 7484 | sgs->nr_numa_running += rq->nr_numa_running; |
| 7485 | sgs->nr_preferred_running += rq->nr_preferred_running; |
| 7486 | #endif |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 7487 | sgs->sum_weighted_load += weighted_cpuload(rq); |
Waiman Long | a426f99 | 2015-11-25 14:09:38 -0500 | [diff] [blame] | 7488 | /* |
| 7489 | * No need to call idle_cpu() if nr_running is not 0 |
| 7490 | */ |
| 7491 | if (!nr_running && idle_cpu(i)) |
Suresh Siddha | aae6d3d | 2010-09-17 15:02:32 -0700 | [diff] [blame] | 7492 | sgs->idle_cpus++; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7493 | } |
| 7494 | |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7495 | /* Adjust by relative CPU capacity of the group */ |
| 7496 | sgs->group_capacity = group->sgc->capacity; |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7497 | sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7498 | |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7499 | if (sgs->sum_nr_running) |
Peter Zijlstra | 38d0f77 | 2013-08-15 19:47:56 +0200 | [diff] [blame] | 7500 | sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7501 | |
Suresh Siddha | aae6d3d | 2010-09-17 15:02:32 -0700 | [diff] [blame] | 7502 | sgs->group_weight = group->group_weight; |
Peter Zijlstra | b37d931 | 2013-08-28 11:50:34 +0200 | [diff] [blame] | 7503 | |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7504 | sgs->group_no_capacity = group_is_overloaded(env, sgs); |
Leo Yan | 79a89f9 | 2015-09-15 18:56:45 +0800 | [diff] [blame] | 7505 | sgs->group_type = group_classify(group, sgs); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7506 | } |
| 7507 | |
| 7508 | /** |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7509 | * update_sd_pick_busiest - return 1 on busiest group |
Randy Dunlap | cd96891 | 2012-06-08 13:18:33 -0700 | [diff] [blame] | 7510 | * @env: The load balancing environment. |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7511 | * @sds: sched_domain statistics |
| 7512 | * @sg: sched_group candidate to be checked for being the busiest |
Michael Neuling | b6b1229 | 2010-06-10 12:06:21 +1000 | [diff] [blame] | 7513 | * @sgs: sched_group statistics |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7514 | * |
| 7515 | * Determine if @sg is a busier group than the previously selected |
| 7516 | * busiest group. |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7517 | * |
| 7518 | * Return: %true if @sg is a busier group than the previously selected |
| 7519 | * busiest group. %false otherwise. |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7520 | */ |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7521 | static bool update_sd_pick_busiest(struct lb_env *env, |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7522 | struct sd_lb_stats *sds, |
| 7523 | struct sched_group *sg, |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7524 | struct sg_lb_stats *sgs) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7525 | { |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7526 | struct sg_lb_stats *busiest = &sds->busiest_stat; |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7527 | |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7528 | if (sgs->group_type > busiest->group_type) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7529 | return true; |
| 7530 | |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7531 | if (sgs->group_type < busiest->group_type) |
| 7532 | return false; |
| 7533 | |
| 7534 | if (sgs->avg_load <= busiest->avg_load) |
| 7535 | return false; |
| 7536 | |
Morten Rasmussen | 9e0994c | 2016-10-14 14:41:10 +0100 | [diff] [blame] | 7537 | if (!(env->sd->flags & SD_ASYM_CPUCAPACITY)) |
| 7538 | goto asym_packing; |
| 7539 | |
| 7540 | /* |
| 7541 | * Candidate sg has no more than one task per CPU and |
| 7542 | * has higher per-CPU capacity. Migrating tasks to less |
| 7543 | * capable CPUs may harm throughput. Maximize throughput, |
| 7544 | * power/energy consequences are not considered. |
| 7545 | */ |
| 7546 | if (sgs->sum_nr_running <= sgs->group_weight && |
| 7547 | group_smaller_cpu_capacity(sds->local, sg)) |
| 7548 | return false; |
| 7549 | |
| 7550 | asym_packing: |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7551 | /* This is the busiest node in its class. */ |
| 7552 | if (!(env->sd->flags & SD_ASYM_PACKING)) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7553 | return true; |
| 7554 | |
Srikar Dronamraju | 1f621e0 | 2016-04-06 18:47:40 +0530 | [diff] [blame] | 7555 | /* No ASYM_PACKING if target cpu is already busy */ |
| 7556 | if (env->idle == CPU_NOT_IDLE) |
| 7557 | return true; |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7558 | /* |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 7559 | * ASYM_PACKING needs to move all the work to the highest |
| 7560 | * prority CPUs in the group, therefore mark all groups |
| 7561 | * of lower priority than ourself as busy. |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7562 | */ |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 7563 | if (sgs->sum_nr_running && |
| 7564 | sched_asym_prefer(env->dst_cpu, sg->asym_prefer_cpu)) { |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7565 | if (!sds->busiest) |
| 7566 | return true; |
| 7567 | |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 7568 | /* Prefer to move from lowest priority cpu's work */ |
| 7569 | if (sched_asym_prefer(sds->busiest->asym_prefer_cpu, |
| 7570 | sg->asym_prefer_cpu)) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7571 | return true; |
| 7572 | } |
| 7573 | |
| 7574 | return false; |
| 7575 | } |
| 7576 | |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 7577 | #ifdef CONFIG_NUMA_BALANCING |
| 7578 | static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) |
| 7579 | { |
| 7580 | if (sgs->sum_nr_running > sgs->nr_numa_running) |
| 7581 | return regular; |
| 7582 | if (sgs->sum_nr_running > sgs->nr_preferred_running) |
| 7583 | return remote; |
| 7584 | return all; |
| 7585 | } |
| 7586 | |
| 7587 | static inline enum fbq_type fbq_classify_rq(struct rq *rq) |
| 7588 | { |
| 7589 | if (rq->nr_running > rq->nr_numa_running) |
| 7590 | return regular; |
| 7591 | if (rq->nr_running > rq->nr_preferred_running) |
| 7592 | return remote; |
| 7593 | return all; |
| 7594 | } |
| 7595 | #else |
| 7596 | static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) |
| 7597 | { |
| 7598 | return all; |
| 7599 | } |
| 7600 | |
| 7601 | static inline enum fbq_type fbq_classify_rq(struct rq *rq) |
| 7602 | { |
| 7603 | return regular; |
| 7604 | } |
| 7605 | #endif /* CONFIG_NUMA_BALANCING */ |
| 7606 | |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7607 | /** |
Hui Kang | 461819a | 2011-10-11 23:00:59 -0400 | [diff] [blame] | 7608 | * update_sd_lb_stats - Update sched_domain's statistics for load balancing. |
Randy Dunlap | cd96891 | 2012-06-08 13:18:33 -0700 | [diff] [blame] | 7609 | * @env: The load balancing environment. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7610 | * @sds: variable to hold the statistics for this sched_domain. |
| 7611 | */ |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 7612 | static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7613 | { |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 7614 | struct sched_domain_shared *shared = env->sd->shared; |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7615 | struct sched_domain *child = env->sd->child; |
| 7616 | struct sched_group *sg = env->sd->groups; |
Srikar Dronamraju | 05b40e0 | 2017-03-22 23:27:50 +0530 | [diff] [blame] | 7617 | struct sg_lb_stats *local = &sds->local_stat; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7618 | struct sg_lb_stats tmp_sgs; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7619 | int load_idx, prefer_sibling = 0; |
Tim Chen | 4486edd | 2014-06-23 12:16:49 -0700 | [diff] [blame] | 7620 | bool overload = false; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7621 | |
| 7622 | if (child && child->flags & SD_PREFER_SIBLING) |
| 7623 | prefer_sibling = 1; |
| 7624 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7625 | load_idx = get_sd_load_idx(env->sd, env->idle); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7626 | |
| 7627 | do { |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7628 | struct sg_lb_stats *sgs = &tmp_sgs; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7629 | int local_group; |
| 7630 | |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 7631 | local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7632 | if (local_group) { |
| 7633 | sds->local = sg; |
Srikar Dronamraju | 05b40e0 | 2017-03-22 23:27:50 +0530 | [diff] [blame] | 7634 | sgs = local; |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7635 | |
| 7636 | if (env->idle != CPU_NEWLY_IDLE || |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7637 | time_after_eq(jiffies, sg->sgc->next_update)) |
| 7638 | update_group_capacity(env->sd, env->dst_cpu); |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7639 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7640 | |
Tim Chen | 4486edd | 2014-06-23 12:16:49 -0700 | [diff] [blame] | 7641 | update_sg_lb_stats(env, sg, load_idx, local_group, sgs, |
| 7642 | &overload); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7643 | |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7644 | if (local_group) |
| 7645 | goto next_group; |
| 7646 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7647 | /* |
| 7648 | * In case the child domain prefers tasks go to siblings |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7649 | * first, lower the sg capacity so that we'll try |
Nikhil Rao | 75dd321 | 2010-10-15 13:12:30 -0700 | [diff] [blame] | 7650 | * and move all the excess tasks away. We lower the capacity |
| 7651 | * of a group only if the local group has the capacity to fit |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7652 | * these excess tasks. The extra check prevents the case where |
| 7653 | * you always pull from the heaviest group when it is already |
| 7654 | * under-utilized (possible with a large weight task outweighs |
| 7655 | * the tasks on the system). |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7656 | */ |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7657 | if (prefer_sibling && sds->local && |
Srikar Dronamraju | 05b40e0 | 2017-03-22 23:27:50 +0530 | [diff] [blame] | 7658 | group_has_capacity(env, local) && |
| 7659 | (sgs->sum_nr_running > local->sum_nr_running + 1)) { |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7660 | sgs->group_no_capacity = 1; |
Leo Yan | 79a89f9 | 2015-09-15 18:56:45 +0800 | [diff] [blame] | 7661 | sgs->group_type = group_classify(sg, sgs); |
Wanpeng Li | cb0b9f2 | 2014-11-05 07:44:50 +0800 | [diff] [blame] | 7662 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7663 | |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7664 | if (update_sd_pick_busiest(env, sds, sg, sgs)) { |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7665 | sds->busiest = sg; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7666 | sds->busiest_stat = *sgs; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7667 | } |
| 7668 | |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7669 | next_group: |
| 7670 | /* Now, start updating sd_lb_stats */ |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 7671 | sds->total_running += sgs->sum_nr_running; |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7672 | sds->total_load += sgs->group_load; |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7673 | sds->total_capacity += sgs->group_capacity; |
Peter Zijlstra | b72ff13 | 2013-08-28 10:32:32 +0200 | [diff] [blame] | 7674 | |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7675 | sg = sg->next; |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7676 | } while (sg != env->sd->groups); |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 7677 | |
| 7678 | if (env->sd->flags & SD_NUMA) |
| 7679 | env->fbq_type = fbq_classify_group(&sds->busiest_stat); |
Tim Chen | 4486edd | 2014-06-23 12:16:49 -0700 | [diff] [blame] | 7680 | |
| 7681 | if (!env->sd->parent) { |
| 7682 | /* update overload indicator if we are at root domain */ |
| 7683 | if (env->dst_rq->rd->overload != overload) |
| 7684 | env->dst_rq->rd->overload = overload; |
| 7685 | } |
| 7686 | |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 7687 | if (!shared) |
| 7688 | return; |
| 7689 | |
| 7690 | /* |
| 7691 | * Since these are sums over groups they can contain some CPUs |
| 7692 | * multiple times for the NUMA domains. |
| 7693 | * |
| 7694 | * Currently only wake_affine_llc() and find_busiest_group() |
| 7695 | * uses these numbers, only the last is affected by this problem. |
| 7696 | * |
| 7697 | * XXX fix that. |
| 7698 | */ |
| 7699 | WRITE_ONCE(shared->nr_running, sds->total_running); |
| 7700 | WRITE_ONCE(shared->load, sds->total_load); |
| 7701 | WRITE_ONCE(shared->capacity, sds->total_capacity); |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7702 | } |
| 7703 | |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7704 | /** |
| 7705 | * check_asym_packing - Check to see if the group is packed into the |
Masanari Iida | 0ba42a5 | 2017-03-07 20:48:02 +0900 | [diff] [blame] | 7706 | * sched domain. |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7707 | * |
| 7708 | * This is primarily intended to used at the sibling level. Some |
| 7709 | * cores like POWER7 prefer to use lower numbered SMT threads. In the |
| 7710 | * case of POWER7, it can move to lower SMT modes only when higher |
| 7711 | * threads are idle. When in lower SMT modes, the threads will |
| 7712 | * perform better since they share less core resources. Hence when we |
| 7713 | * have idle threads, we want them to be the higher ones. |
| 7714 | * |
| 7715 | * This packing function is run on idle threads. It checks to see if |
| 7716 | * the busiest CPU in this domain (core in the P7 case) has a higher |
| 7717 | * CPU number than the packing function is being run on. Here we are |
| 7718 | * assuming lower CPU number will be equivalent to lower a SMT thread |
| 7719 | * number. |
| 7720 | * |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7721 | * Return: 1 when packing is required and a task should be moved to |
Michael Neuling | b6b1229 | 2010-06-10 12:06:21 +1000 | [diff] [blame] | 7722 | * this CPU. The amount of the imbalance is returned in *imbalance. |
| 7723 | * |
Randy Dunlap | cd96891 | 2012-06-08 13:18:33 -0700 | [diff] [blame] | 7724 | * @env: The load balancing environment. |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7725 | * @sds: Statistics of the sched_domain which is to be packed |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7726 | */ |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7727 | static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7728 | { |
| 7729 | int busiest_cpu; |
| 7730 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7731 | if (!(env->sd->flags & SD_ASYM_PACKING)) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7732 | return 0; |
| 7733 | |
Srikar Dronamraju | 1f621e0 | 2016-04-06 18:47:40 +0530 | [diff] [blame] | 7734 | if (env->idle == CPU_NOT_IDLE) |
| 7735 | return 0; |
| 7736 | |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7737 | if (!sds->busiest) |
| 7738 | return 0; |
| 7739 | |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 7740 | busiest_cpu = sds->busiest->asym_prefer_cpu; |
| 7741 | if (sched_asym_prefer(busiest_cpu, env->dst_cpu)) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7742 | return 0; |
| 7743 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7744 | env->imbalance = DIV_ROUND_CLOSEST( |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7745 | sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity, |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7746 | SCHED_CAPACITY_SCALE); |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7747 | |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7748 | return 1; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7749 | } |
| 7750 | |
| 7751 | /** |
| 7752 | * fix_small_imbalance - Calculate the minor imbalance that exists |
| 7753 | * amongst the groups of a sched_domain, during |
| 7754 | * load balancing. |
Randy Dunlap | cd96891 | 2012-06-08 13:18:33 -0700 | [diff] [blame] | 7755 | * @env: The load balancing environment. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7756 | * @sds: Statistics of the sched_domain whose imbalance is to be calculated. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7757 | */ |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7758 | static inline |
| 7759 | void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7760 | { |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7761 | unsigned long tmp, capa_now = 0, capa_move = 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7762 | unsigned int imbn = 2; |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7763 | unsigned long scaled_busy_load_per_task; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7764 | struct sg_lb_stats *local, *busiest; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7765 | |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7766 | local = &sds->local_stat; |
| 7767 | busiest = &sds->busiest_stat; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7768 | |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7769 | if (!local->sum_nr_running) |
| 7770 | local->load_per_task = cpu_avg_load_per_task(env->dst_cpu); |
| 7771 | else if (busiest->load_per_task > local->load_per_task) |
| 7772 | imbn = 1; |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7773 | |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7774 | scaled_busy_load_per_task = |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7775 | (busiest->load_per_task * SCHED_CAPACITY_SCALE) / |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7776 | busiest->group_capacity; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7777 | |
Vladimir Davydov | 3029ede | 2013-09-15 17:49:14 +0400 | [diff] [blame] | 7778 | if (busiest->avg_load + scaled_busy_load_per_task >= |
| 7779 | local->avg_load + (scaled_busy_load_per_task * imbn)) { |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7780 | env->imbalance = busiest->load_per_task; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7781 | return; |
| 7782 | } |
| 7783 | |
| 7784 | /* |
| 7785 | * OK, we don't have enough imbalance to justify moving tasks, |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 7786 | * however we may be able to increase total CPU capacity used by |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7787 | * moving them. |
| 7788 | */ |
| 7789 | |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7790 | capa_now += busiest->group_capacity * |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7791 | min(busiest->load_per_task, busiest->avg_load); |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7792 | capa_now += local->group_capacity * |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7793 | min(local->load_per_task, local->avg_load); |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7794 | capa_now /= SCHED_CAPACITY_SCALE; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7795 | |
| 7796 | /* Amount of load we'd subtract */ |
Vincent Guittot | a2cd426 | 2014-03-11 17:26:06 +0100 | [diff] [blame] | 7797 | if (busiest->avg_load > scaled_busy_load_per_task) { |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7798 | capa_move += busiest->group_capacity * |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7799 | min(busiest->load_per_task, |
Vincent Guittot | a2cd426 | 2014-03-11 17:26:06 +0100 | [diff] [blame] | 7800 | busiest->avg_load - scaled_busy_load_per_task); |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7801 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7802 | |
| 7803 | /* Amount of load we'd add */ |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7804 | if (busiest->avg_load * busiest->group_capacity < |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7805 | busiest->load_per_task * SCHED_CAPACITY_SCALE) { |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7806 | tmp = (busiest->avg_load * busiest->group_capacity) / |
| 7807 | local->group_capacity; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7808 | } else { |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7809 | tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) / |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7810 | local->group_capacity; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7811 | } |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7812 | capa_move += local->group_capacity * |
Peter Zijlstra | 3ae11c9 | 2013-08-15 20:37:48 +0200 | [diff] [blame] | 7813 | min(local->load_per_task, local->avg_load + tmp); |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7814 | capa_move /= SCHED_CAPACITY_SCALE; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7815 | |
| 7816 | /* Move if we gain throughput */ |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7817 | if (capa_move > capa_now) |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7818 | env->imbalance = busiest->load_per_task; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7819 | } |
| 7820 | |
| 7821 | /** |
| 7822 | * calculate_imbalance - Calculate the amount of imbalance present within the |
| 7823 | * groups of a given sched_domain during load balance. |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7824 | * @env: load balance environment |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7825 | * @sds: statistics of the sched_domain whose imbalance is to be calculated. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7826 | */ |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7827 | static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7828 | { |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7829 | unsigned long max_pull, load_above_capacity = ~0UL; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7830 | struct sg_lb_stats *local, *busiest; |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7831 | |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7832 | local = &sds->local_stat; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7833 | busiest = &sds->busiest_stat; |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7834 | |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7835 | if (busiest->group_type == group_imbalanced) { |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7836 | /* |
| 7837 | * In the group_imb case we cannot rely on group-wide averages |
| 7838 | * to ensure cpu-load equilibrium, look at wider averages. XXX |
| 7839 | */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7840 | busiest->load_per_task = |
| 7841 | min(busiest->load_per_task, sds->avg_load); |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7842 | } |
| 7843 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7844 | /* |
Dietmar Eggemann | 885e542 | 2016-04-29 20:32:39 +0100 | [diff] [blame] | 7845 | * Avg load of busiest sg can be less and avg load of local sg can |
| 7846 | * be greater than avg load across all sgs of sd because avg load |
| 7847 | * factors in sg capacity and sgs with smaller group_type are |
| 7848 | * skipped when updating the busiest sg: |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7849 | */ |
Vladimir Davydov | b188555 | 2013-09-15 17:49:13 +0400 | [diff] [blame] | 7850 | if (busiest->avg_load <= sds->avg_load || |
| 7851 | local->avg_load >= sds->avg_load) { |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7852 | env->imbalance = 0; |
| 7853 | return fix_small_imbalance(env, sds); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7854 | } |
| 7855 | |
Peter Zijlstra | 9a5d9ba | 2014-07-29 17:15:11 +0200 | [diff] [blame] | 7856 | /* |
| 7857 | * If there aren't any idle cpus, avoid creating some. |
| 7858 | */ |
| 7859 | if (busiest->group_type == group_overloaded && |
| 7860 | local->group_type == group_overloaded) { |
Peter Zijlstra | 1be0eb2 | 2016-05-06 12:21:23 +0200 | [diff] [blame] | 7861 | load_above_capacity = busiest->sum_nr_running * SCHED_CAPACITY_SCALE; |
Morten Rasmussen | cfa1033 | 2016-04-29 20:32:40 +0100 | [diff] [blame] | 7862 | if (load_above_capacity > busiest->group_capacity) { |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7863 | load_above_capacity -= busiest->group_capacity; |
Dietmar Eggemann | 2665621 | 2016-08-10 11:27:27 +0100 | [diff] [blame] | 7864 | load_above_capacity *= scale_load_down(NICE_0_LOAD); |
Morten Rasmussen | cfa1033 | 2016-04-29 20:32:40 +0100 | [diff] [blame] | 7865 | load_above_capacity /= busiest->group_capacity; |
| 7866 | } else |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7867 | load_above_capacity = ~0UL; |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7868 | } |
| 7869 | |
| 7870 | /* |
| 7871 | * We're trying to get all the cpus to the average_load, so we don't |
| 7872 | * want to push ourselves above the average load, nor do we wish to |
| 7873 | * reduce the max loaded cpu below the average load. At the same time, |
Dietmar Eggemann | 0a9b23c | 2016-04-29 20:32:38 +0100 | [diff] [blame] | 7874 | * we also don't want to reduce the group load below the group |
| 7875 | * capacity. Thus we look for the minimum possible imbalance. |
Suresh Siddha | dd5feea | 2010-02-23 16:13:52 -0800 | [diff] [blame] | 7876 | */ |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7877 | max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7878 | |
| 7879 | /* How much load to actually move to equalise the imbalance */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7880 | env->imbalance = min( |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 7881 | max_pull * busiest->group_capacity, |
| 7882 | (sds->avg_load - local->avg_load) * local->group_capacity |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7883 | ) / SCHED_CAPACITY_SCALE; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7884 | |
| 7885 | /* |
| 7886 | * if *imbalance is less than the average load per runnable task |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 7887 | * there is no guarantee that any tasks will be moved so we'll have |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7888 | * a think about bumping its value to force at least one task to be |
| 7889 | * moved |
| 7890 | */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7891 | if (env->imbalance < busiest->load_per_task) |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7892 | return fix_small_imbalance(env, sds); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7893 | } |
Nikhil Rao | fab4762 | 2010-10-15 13:12:29 -0700 | [diff] [blame] | 7894 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7895 | /******* find_busiest_group() helpers end here *********************/ |
| 7896 | |
| 7897 | /** |
| 7898 | * find_busiest_group - Returns the busiest group within the sched_domain |
Dietmar Eggemann | 0a9b23c | 2016-04-29 20:32:38 +0100 | [diff] [blame] | 7899 | * if there is an imbalance. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7900 | * |
| 7901 | * Also calculates the amount of weighted load which should be moved |
| 7902 | * to restore balance. |
| 7903 | * |
Randy Dunlap | cd96891 | 2012-06-08 13:18:33 -0700 | [diff] [blame] | 7904 | * @env: The load balancing environment. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7905 | * |
Yacine Belkadi | e69f618 | 2013-07-12 20:45:47 +0200 | [diff] [blame] | 7906 | * Return: - The busiest group if imbalance exists. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7907 | */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7908 | static struct sched_group *find_busiest_group(struct lb_env *env) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7909 | { |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7910 | struct sg_lb_stats *local, *busiest; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7911 | struct sd_lb_stats sds; |
| 7912 | |
Peter Zijlstra | 147c5fc | 2013-08-19 15:22:57 +0200 | [diff] [blame] | 7913 | init_sd_lb_stats(&sds); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7914 | |
| 7915 | /* |
| 7916 | * Compute the various statistics relavent for load balancing at |
| 7917 | * this level. |
| 7918 | */ |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 7919 | update_sd_lb_stats(env, &sds); |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7920 | local = &sds.local_stat; |
| 7921 | busiest = &sds.busiest_stat; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7922 | |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7923 | /* ASYM feature bypasses nice load balance check */ |
Srikar Dronamraju | 1f621e0 | 2016-04-06 18:47:40 +0530 | [diff] [blame] | 7924 | if (check_asym_packing(env, &sds)) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 7925 | return sds.busiest; |
| 7926 | |
Peter Zijlstra | cc57aa8 | 2011-02-21 18:55:32 +0100 | [diff] [blame] | 7927 | /* There is no busy sibling group to pull tasks from */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7928 | if (!sds.busiest || busiest->sum_nr_running == 0) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7929 | goto out_balanced; |
| 7930 | |
Peter Zijlstra | 90001d6 | 2017-07-31 17:50:05 +0200 | [diff] [blame^] | 7931 | /* XXX broken for overlapping NUMA groups */ |
Nicolas Pitre | ca8ce3d | 2014-05-26 18:19:39 -0400 | [diff] [blame] | 7932 | sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) |
| 7933 | / sds.total_capacity; |
Ken Chen | b0432d8 | 2011-04-07 17:23:22 -0700 | [diff] [blame] | 7934 | |
Peter Zijlstra | 866ab43 | 2011-02-21 18:56:47 +0100 | [diff] [blame] | 7935 | /* |
| 7936 | * If the busiest group is imbalanced the below checks don't |
Peter Zijlstra | 30ce5da | 2013-08-15 20:29:29 +0200 | [diff] [blame] | 7937 | * work because they assume all things are equal, which typically |
Peter Zijlstra | 866ab43 | 2011-02-21 18:56:47 +0100 | [diff] [blame] | 7938 | * isn't true due to cpus_allowed constraints and the like. |
| 7939 | */ |
Rik van Riel | caeb178 | 2014-07-28 14:16:28 -0400 | [diff] [blame] | 7940 | if (busiest->group_type == group_imbalanced) |
Peter Zijlstra | 866ab43 | 2011-02-21 18:56:47 +0100 | [diff] [blame] | 7941 | goto force_balance; |
| 7942 | |
Peter Zijlstra | cc57aa8 | 2011-02-21 18:55:32 +0100 | [diff] [blame] | 7943 | /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */ |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 7944 | if (env->idle == CPU_NEWLY_IDLE && group_has_capacity(env, local) && |
| 7945 | busiest->group_no_capacity) |
Nikhil Rao | fab4762 | 2010-10-15 13:12:29 -0700 | [diff] [blame] | 7946 | goto force_balance; |
| 7947 | |
Peter Zijlstra | cc57aa8 | 2011-02-21 18:55:32 +0100 | [diff] [blame] | 7948 | /* |
Zhihui Zhang | 9c58c79 | 2014-09-20 21:24:36 -0400 | [diff] [blame] | 7949 | * If the local group is busier than the selected busiest group |
Peter Zijlstra | cc57aa8 | 2011-02-21 18:55:32 +0100 | [diff] [blame] | 7950 | * don't try and pull any tasks. |
| 7951 | */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7952 | if (local->avg_load >= busiest->avg_load) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7953 | goto out_balanced; |
| 7954 | |
Peter Zijlstra | cc57aa8 | 2011-02-21 18:55:32 +0100 | [diff] [blame] | 7955 | /* |
| 7956 | * Don't pull any tasks if this group is already above the domain |
| 7957 | * average load. |
| 7958 | */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7959 | if (local->avg_load >= sds.avg_load) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7960 | goto out_balanced; |
| 7961 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7962 | if (env->idle == CPU_IDLE) { |
Suresh Siddha | aae6d3d | 2010-09-17 15:02:32 -0700 | [diff] [blame] | 7963 | /* |
Vincent Guittot | 43f4d66 | 2014-10-01 15:38:55 +0200 | [diff] [blame] | 7964 | * This cpu is idle. If the busiest group is not overloaded |
| 7965 | * and there is no imbalance between this and busiest group |
| 7966 | * wrt idle cpus, it is balanced. The imbalance becomes |
| 7967 | * significant if the diff is greater than 1 otherwise we |
| 7968 | * might end up to just move the imbalance on another group |
Suresh Siddha | aae6d3d | 2010-09-17 15:02:32 -0700 | [diff] [blame] | 7969 | */ |
Vincent Guittot | 43f4d66 | 2014-10-01 15:38:55 +0200 | [diff] [blame] | 7970 | if ((busiest->group_type != group_overloaded) && |
| 7971 | (local->idle_cpus <= (busiest->idle_cpus + 1))) |
Suresh Siddha | aae6d3d | 2010-09-17 15:02:32 -0700 | [diff] [blame] | 7972 | goto out_balanced; |
Peter Zijlstra | c186faf | 2011-02-21 18:52:53 +0100 | [diff] [blame] | 7973 | } else { |
| 7974 | /* |
| 7975 | * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use |
| 7976 | * imbalance_pct to be conservative. |
| 7977 | */ |
Joonsoo Kim | 56cf515 | 2013-08-06 17:36:43 +0900 | [diff] [blame] | 7978 | if (100 * busiest->avg_load <= |
| 7979 | env->sd->imbalance_pct * local->avg_load) |
Peter Zijlstra | c186faf | 2011-02-21 18:52:53 +0100 | [diff] [blame] | 7980 | goto out_balanced; |
Suresh Siddha | aae6d3d | 2010-09-17 15:02:32 -0700 | [diff] [blame] | 7981 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7982 | |
Nikhil Rao | fab4762 | 2010-10-15 13:12:29 -0700 | [diff] [blame] | 7983 | force_balance: |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7984 | /* Looks like there is an imbalance. Compute it */ |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7985 | calculate_imbalance(env, &sds); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7986 | return sds.busiest; |
| 7987 | |
| 7988 | out_balanced: |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7989 | env->imbalance = 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7990 | return NULL; |
| 7991 | } |
| 7992 | |
| 7993 | /* |
| 7994 | * find_busiest_queue - find the busiest runqueue among the cpus in group. |
| 7995 | */ |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 7996 | static struct rq *find_busiest_queue(struct lb_env *env, |
Michael Wang | b9403130 | 2012-07-12 16:10:13 +0800 | [diff] [blame] | 7997 | struct sched_group *group) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 7998 | { |
| 7999 | struct rq *busiest = NULL, *rq; |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8000 | unsigned long busiest_load = 0, busiest_capacity = 1; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8001 | int i; |
| 8002 | |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 8003 | for_each_cpu_and(i, sched_group_span(group), env->cpus) { |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 8004 | unsigned long capacity, wl; |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 8005 | enum fbq_type rt; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8006 | |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 8007 | rq = cpu_rq(i); |
| 8008 | rt = fbq_classify_rq(rq); |
| 8009 | |
| 8010 | /* |
| 8011 | * We classify groups/runqueues into three groups: |
| 8012 | * - regular: there are !numa tasks |
| 8013 | * - remote: there are numa tasks that run on the 'wrong' node |
| 8014 | * - all: there is no distinction |
| 8015 | * |
| 8016 | * In order to avoid migrating ideally placed numa tasks, |
| 8017 | * ignore those when there's better options. |
| 8018 | * |
| 8019 | * If we ignore the actual busiest queue to migrate another |
| 8020 | * task, the next balance pass can still reduce the busiest |
| 8021 | * queue by moving tasks around inside the node. |
| 8022 | * |
| 8023 | * If we cannot move enough load due to this classification |
| 8024 | * the next pass will adjust the group classification and |
| 8025 | * allow migration of more tasks. |
| 8026 | * |
| 8027 | * Both cases only affect the total convergence complexity. |
| 8028 | */ |
| 8029 | if (rt > env->fbq_type) |
| 8030 | continue; |
| 8031 | |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8032 | capacity = capacity_of(i); |
Srivatsa Vaddagiri | 9d5efe0 | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 8033 | |
Viresh Kumar | c7132dd | 2017-05-24 10:59:54 +0530 | [diff] [blame] | 8034 | wl = weighted_cpuload(rq); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8035 | |
Thomas Gleixner | 6e40f5b | 2010-02-16 16:48:56 +0100 | [diff] [blame] | 8036 | /* |
| 8037 | * When comparing with imbalance, use weighted_cpuload() |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8038 | * which is not scaled with the cpu capacity. |
Thomas Gleixner | 6e40f5b | 2010-02-16 16:48:56 +0100 | [diff] [blame] | 8039 | */ |
Vincent Guittot | ea67821 | 2015-02-27 16:54:11 +0100 | [diff] [blame] | 8040 | |
| 8041 | if (rq->nr_running == 1 && wl > env->imbalance && |
| 8042 | !check_cpu_capacity(rq, env->sd)) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8043 | continue; |
| 8044 | |
Thomas Gleixner | 6e40f5b | 2010-02-16 16:48:56 +0100 | [diff] [blame] | 8045 | /* |
| 8046 | * For the load comparisons with the other cpu's, consider |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8047 | * the weighted_cpuload() scaled with the cpu capacity, so |
| 8048 | * that the load can be moved away from the cpu that is |
| 8049 | * potentially running at a lower capacity. |
Joonsoo Kim | 95a79b8 | 2013-08-06 17:36:41 +0900 | [diff] [blame] | 8050 | * |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8051 | * Thus we're looking for max(wl_i / capacity_i), crosswise |
Joonsoo Kim | 95a79b8 | 2013-08-06 17:36:41 +0900 | [diff] [blame] | 8052 | * multiplication to rid ourselves of the division works out |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8053 | * to: wl_i * capacity_j > wl_j * capacity_i; where j is |
| 8054 | * our previous maximum. |
Thomas Gleixner | 6e40f5b | 2010-02-16 16:48:56 +0100 | [diff] [blame] | 8055 | */ |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8056 | if (wl * busiest_capacity > busiest_load * capacity) { |
Joonsoo Kim | 95a79b8 | 2013-08-06 17:36:41 +0900 | [diff] [blame] | 8057 | busiest_load = wl; |
Nicolas Pitre | ced549f | 2014-05-26 18:19:38 -0400 | [diff] [blame] | 8058 | busiest_capacity = capacity; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8059 | busiest = rq; |
| 8060 | } |
| 8061 | } |
| 8062 | |
| 8063 | return busiest; |
| 8064 | } |
| 8065 | |
| 8066 | /* |
| 8067 | * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but |
| 8068 | * so long as it is large enough. |
| 8069 | */ |
| 8070 | #define MAX_PINNED_INTERVAL 512 |
| 8071 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 8072 | static int need_active_balance(struct lb_env *env) |
Peter Zijlstra | 1af3ed3 | 2009-12-23 15:10:31 +0100 | [diff] [blame] | 8073 | { |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 8074 | struct sched_domain *sd = env->sd; |
| 8075 | |
| 8076 | if (env->idle == CPU_NEWLY_IDLE) { |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 8077 | |
| 8078 | /* |
| 8079 | * ASYM_PACKING needs to force migrate tasks from busy but |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 8080 | * lower priority CPUs in order to pack all tasks in the |
| 8081 | * highest priority CPUs. |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 8082 | */ |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 8083 | if ((sd->flags & SD_ASYM_PACKING) && |
| 8084 | sched_asym_prefer(env->dst_cpu, env->src_cpu)) |
Michael Neuling | 532cb4c | 2010-06-08 14:57:02 +1000 | [diff] [blame] | 8085 | return 1; |
Peter Zijlstra | 1af3ed3 | 2009-12-23 15:10:31 +0100 | [diff] [blame] | 8086 | } |
| 8087 | |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8088 | /* |
| 8089 | * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. |
| 8090 | * It's worth migrating the task if the src_cpu's capacity is reduced |
| 8091 | * because of other sched_class or IRQs if more capacity stays |
| 8092 | * available on dst_cpu. |
| 8093 | */ |
| 8094 | if ((env->idle != CPU_NOT_IDLE) && |
| 8095 | (env->src_rq->cfs.h_nr_running == 1)) { |
| 8096 | if ((check_cpu_capacity(env->src_rq, sd)) && |
| 8097 | (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) |
| 8098 | return 1; |
| 8099 | } |
| 8100 | |
Peter Zijlstra | 1af3ed3 | 2009-12-23 15:10:31 +0100 | [diff] [blame] | 8101 | return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); |
| 8102 | } |
| 8103 | |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8104 | static int active_load_balance_cpu_stop(void *data); |
| 8105 | |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8106 | static int should_we_balance(struct lb_env *env) |
| 8107 | { |
| 8108 | struct sched_group *sg = env->sd->groups; |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8109 | int cpu, balance_cpu = -1; |
| 8110 | |
| 8111 | /* |
| 8112 | * In the newly idle case, we will allow all the cpu's |
| 8113 | * to do the newly idle load balance. |
| 8114 | */ |
| 8115 | if (env->idle == CPU_NEWLY_IDLE) |
| 8116 | return 1; |
| 8117 | |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8118 | /* Try to find first idle cpu */ |
Peter Zijlstra | e5c14b1 | 2017-05-01 10:47:02 +0200 | [diff] [blame] | 8119 | for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) { |
Peter Zijlstra | af21812 | 2017-05-01 08:51:05 +0200 | [diff] [blame] | 8120 | if (!idle_cpu(cpu)) |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8121 | continue; |
| 8122 | |
| 8123 | balance_cpu = cpu; |
| 8124 | break; |
| 8125 | } |
| 8126 | |
| 8127 | if (balance_cpu == -1) |
| 8128 | balance_cpu = group_balance_cpu(sg); |
| 8129 | |
| 8130 | /* |
| 8131 | * First idle cpu or the first cpu(busiest) in this sched group |
| 8132 | * is eligible for doing load balancing at this and above domains. |
| 8133 | */ |
Joonsoo Kim | b0cff9d | 2013-09-10 15:54:49 +0900 | [diff] [blame] | 8134 | return balance_cpu == env->dst_cpu; |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8135 | } |
| 8136 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8137 | /* |
| 8138 | * Check this_cpu to ensure it is balanced within domain. Attempt to move |
| 8139 | * tasks if there is an imbalance. |
| 8140 | */ |
| 8141 | static int load_balance(int this_cpu, struct rq *this_rq, |
| 8142 | struct sched_domain *sd, enum cpu_idle_type idle, |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8143 | int *continue_balancing) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8144 | { |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8145 | int ld_moved, cur_ld_moved, active_balance = 0; |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8146 | struct sched_domain *sd_parent = sd->parent; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8147 | struct sched_group *group; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8148 | struct rq *busiest; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8149 | struct rq_flags rf; |
Christoph Lameter | 4ba2968 | 2014-08-26 19:12:21 -0500 | [diff] [blame] | 8150 | struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8151 | |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8152 | struct lb_env env = { |
| 8153 | .sd = sd, |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 8154 | .dst_cpu = this_cpu, |
| 8155 | .dst_rq = this_rq, |
Peter Zijlstra | ae4df9d | 2017-05-01 11:03:12 +0200 | [diff] [blame] | 8156 | .dst_grpmask = sched_group_span(sd->groups), |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8157 | .idle = idle, |
Peter Zijlstra | eb95308 | 2012-04-17 13:38:40 +0200 | [diff] [blame] | 8158 | .loop_break = sched_nr_migrate_break, |
Michael Wang | b9403130 | 2012-07-12 16:10:13 +0800 | [diff] [blame] | 8159 | .cpus = cpus, |
Peter Zijlstra | 0ec8aa0 | 2013-10-07 11:29:33 +0100 | [diff] [blame] | 8160 | .fbq_type = all, |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 8161 | .tasks = LIST_HEAD_INIT(env.tasks), |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8162 | }; |
| 8163 | |
Jeffrey Hugo | 65a4433 | 2017-06-07 13:18:57 -0600 | [diff] [blame] | 8164 | cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8165 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8166 | schedstat_inc(sd->lb_count[idle]); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8167 | |
| 8168 | redo: |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8169 | if (!should_we_balance(&env)) { |
| 8170 | *continue_balancing = 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8171 | goto out_balanced; |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8172 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8173 | |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8174 | group = find_busiest_group(&env); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8175 | if (!group) { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8176 | schedstat_inc(sd->lb_nobusyg[idle]); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8177 | goto out_balanced; |
| 8178 | } |
| 8179 | |
Michael Wang | b9403130 | 2012-07-12 16:10:13 +0800 | [diff] [blame] | 8180 | busiest = find_busiest_queue(&env, group); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8181 | if (!busiest) { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8182 | schedstat_inc(sd->lb_nobusyq[idle]); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8183 | goto out_balanced; |
| 8184 | } |
| 8185 | |
Michael Wang | 78feefc | 2012-08-06 16:41:59 +0800 | [diff] [blame] | 8186 | BUG_ON(busiest == env.dst_rq); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8187 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8188 | schedstat_add(sd->lb_imbalance[idle], env.imbalance); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8189 | |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8190 | env.src_cpu = busiest->cpu; |
| 8191 | env.src_rq = busiest; |
| 8192 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8193 | ld_moved = 0; |
| 8194 | if (busiest->nr_running > 1) { |
| 8195 | /* |
| 8196 | * Attempt to move tasks. If find_busiest_group has found |
| 8197 | * an imbalance but busiest->nr_running <= 1, the group is |
| 8198 | * still unbalanced. ld_moved simply stays zero, so it is |
| 8199 | * correctly treated as an imbalance. |
| 8200 | */ |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8201 | env.flags |= LBF_ALL_PINNED; |
Peter Zijlstra | c82513e | 2012-04-26 13:12:27 +0200 | [diff] [blame] | 8202 | env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8203 | |
Peter Zijlstra | 5d6523e | 2012-03-10 00:07:36 +0100 | [diff] [blame] | 8204 | more_balance: |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8205 | rq_lock_irqsave(busiest, &rf); |
Peter Zijlstra | 3bed5e2 | 2016-10-03 16:35:32 +0200 | [diff] [blame] | 8206 | update_rq_clock(busiest); |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8207 | |
| 8208 | /* |
| 8209 | * cur_ld_moved - load moved in current iteration |
| 8210 | * ld_moved - cumulative load moved across iterations |
| 8211 | */ |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 8212 | cur_ld_moved = detach_tasks(&env); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8213 | |
| 8214 | /* |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 8215 | * We've detached some tasks from busiest_rq. Every |
| 8216 | * task is masked "TASK_ON_RQ_MIGRATING", so we can safely |
| 8217 | * unlock busiest->lock, and we are able to be sure |
| 8218 | * that nobody can manipulate the tasks in parallel. |
| 8219 | * See task_rq_lock() family for the details. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8220 | */ |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 8221 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8222 | rq_unlock(busiest, &rf); |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 8223 | |
| 8224 | if (cur_ld_moved) { |
| 8225 | attach_tasks(&env); |
| 8226 | ld_moved += cur_ld_moved; |
| 8227 | } |
| 8228 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8229 | local_irq_restore(rf.flags); |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8230 | |
Joonsoo Kim | f1cd085 | 2013-04-23 17:27:37 +0900 | [diff] [blame] | 8231 | if (env.flags & LBF_NEED_BREAK) { |
| 8232 | env.flags &= ~LBF_NEED_BREAK; |
| 8233 | goto more_balance; |
| 8234 | } |
| 8235 | |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8236 | /* |
| 8237 | * Revisit (affine) tasks on src_cpu that couldn't be moved to |
| 8238 | * us and move them to an alternate dst_cpu in our sched_group |
| 8239 | * where they can run. The upper limit on how many times we |
| 8240 | * iterate on same src_cpu is dependent on number of cpus in our |
| 8241 | * sched_group. |
| 8242 | * |
| 8243 | * This changes load balance semantics a bit on who can move |
| 8244 | * load to a given_cpu. In addition to the given_cpu itself |
| 8245 | * (or a ilb_cpu acting on its behalf where given_cpu is |
| 8246 | * nohz-idle), we now have balance_cpu in a position to move |
| 8247 | * load to given_cpu. In rare situations, this may cause |
| 8248 | * conflicts (balance_cpu and given_cpu/ilb_cpu deciding |
| 8249 | * _independently_ and at _same_ time to move some load to |
| 8250 | * given_cpu) causing exceess load to be moved to given_cpu. |
| 8251 | * This however should not happen so much in practice and |
| 8252 | * moreover subsequent load balance cycles should correct the |
| 8253 | * excess load moved. |
| 8254 | */ |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8255 | if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8256 | |
Vladimir Davydov | 7aff2e3 | 2013-09-15 21:30:13 +0400 | [diff] [blame] | 8257 | /* Prevent to re-select dst_cpu via env's cpus */ |
| 8258 | cpumask_clear_cpu(env.dst_cpu, env.cpus); |
| 8259 | |
Michael Wang | 78feefc | 2012-08-06 16:41:59 +0800 | [diff] [blame] | 8260 | env.dst_rq = cpu_rq(env.new_dst_cpu); |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8261 | env.dst_cpu = env.new_dst_cpu; |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8262 | env.flags &= ~LBF_DST_PINNED; |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8263 | env.loop = 0; |
| 8264 | env.loop_break = sched_nr_migrate_break; |
Joonsoo Kim | e02e60c | 2013-04-23 17:27:42 +0900 | [diff] [blame] | 8265 | |
Srivatsa Vaddagiri | 88b8dac | 2012-06-19 17:43:15 +0530 | [diff] [blame] | 8266 | /* |
| 8267 | * Go back to "more_balance" rather than "redo" since we |
| 8268 | * need to continue with same src_cpu. |
| 8269 | */ |
| 8270 | goto more_balance; |
| 8271 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8272 | |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8273 | /* |
| 8274 | * We failed to reach balance because of affinity. |
| 8275 | */ |
| 8276 | if (sd_parent) { |
Nicolas Pitre | 63b2ca3 | 2014-05-26 18:19:37 -0400 | [diff] [blame] | 8277 | int *group_imbalance = &sd_parent->groups->sgc->imbalance; |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8278 | |
Vincent Guittot | afdeee0 | 2014-08-26 13:06:44 +0200 | [diff] [blame] | 8279 | if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8280 | *group_imbalance = 1; |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8281 | } |
| 8282 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8283 | /* All tasks on this runqueue were pinned by CPU affinity */ |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8284 | if (unlikely(env.flags & LBF_ALL_PINNED)) { |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8285 | cpumask_clear_cpu(cpu_of(busiest), cpus); |
Jeffrey Hugo | 65a4433 | 2017-06-07 13:18:57 -0600 | [diff] [blame] | 8286 | /* |
| 8287 | * Attempting to continue load balancing at the current |
| 8288 | * sched_domain level only makes sense if there are |
| 8289 | * active CPUs remaining as possible busiest CPUs to |
| 8290 | * pull load from which are not contained within the |
| 8291 | * destination group that is receiving any migrated |
| 8292 | * load. |
| 8293 | */ |
| 8294 | if (!cpumask_subset(cpus, env.dst_grpmask)) { |
Prashanth Nageshappa | bbf18b1 | 2012-06-19 17:52:07 +0530 | [diff] [blame] | 8295 | env.loop = 0; |
| 8296 | env.loop_break = sched_nr_migrate_break; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8297 | goto redo; |
Prashanth Nageshappa | bbf18b1 | 2012-06-19 17:52:07 +0530 | [diff] [blame] | 8298 | } |
Vincent Guittot | afdeee0 | 2014-08-26 13:06:44 +0200 | [diff] [blame] | 8299 | goto out_all_pinned; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8300 | } |
| 8301 | } |
| 8302 | |
| 8303 | if (!ld_moved) { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8304 | schedstat_inc(sd->lb_failed[idle]); |
Venkatesh Pallipadi | 58b26c4 | 2010-09-10 18:19:17 -0700 | [diff] [blame] | 8305 | /* |
| 8306 | * Increment the failure counter only on periodic balance. |
| 8307 | * We do not want newidle balance, which can be very |
| 8308 | * frequent, pollute the failure counter causing |
| 8309 | * excessive cache_hot migrations and active balances. |
| 8310 | */ |
| 8311 | if (idle != CPU_NEWLY_IDLE) |
| 8312 | sd->nr_balance_failed++; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8313 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 8314 | if (need_active_balance(&env)) { |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8315 | unsigned long flags; |
| 8316 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8317 | raw_spin_lock_irqsave(&busiest->lock, flags); |
| 8318 | |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8319 | /* don't kick the active_load_balance_cpu_stop, |
| 8320 | * if the curr task on busiest cpu can't be |
| 8321 | * moved to this_cpu |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8322 | */ |
Ingo Molnar | 0c98d34 | 2017-02-05 15:38:10 +0100 | [diff] [blame] | 8323 | if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) { |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8324 | raw_spin_unlock_irqrestore(&busiest->lock, |
| 8325 | flags); |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8326 | env.flags |= LBF_ALL_PINNED; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8327 | goto out_one_pinned; |
| 8328 | } |
| 8329 | |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8330 | /* |
| 8331 | * ->active_balance synchronizes accesses to |
| 8332 | * ->active_balance_work. Once set, it's cleared |
| 8333 | * only after active load balance is finished. |
| 8334 | */ |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8335 | if (!busiest->active_balance) { |
| 8336 | busiest->active_balance = 1; |
| 8337 | busiest->push_cpu = this_cpu; |
| 8338 | active_balance = 1; |
| 8339 | } |
| 8340 | raw_spin_unlock_irqrestore(&busiest->lock, flags); |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8341 | |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 8342 | if (active_balance) { |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8343 | stop_one_cpu_nowait(cpu_of(busiest), |
| 8344 | active_load_balance_cpu_stop, busiest, |
| 8345 | &busiest->active_balance_work); |
Peter Zijlstra | bd939f4 | 2012-05-02 14:20:37 +0200 | [diff] [blame] | 8346 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8347 | |
Srikar Dronamraju | d02c07118 | 2016-03-23 17:54:44 +0530 | [diff] [blame] | 8348 | /* We've kicked active balancing, force task migration. */ |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8349 | sd->nr_balance_failed = sd->cache_nice_tries+1; |
| 8350 | } |
| 8351 | } else |
| 8352 | sd->nr_balance_failed = 0; |
| 8353 | |
| 8354 | if (likely(!active_balance)) { |
| 8355 | /* We were unbalanced, so reset the balancing interval */ |
| 8356 | sd->balance_interval = sd->min_interval; |
| 8357 | } else { |
| 8358 | /* |
| 8359 | * If we've begun active balancing, start to back off. This |
| 8360 | * case may not be covered by the all_pinned logic if there |
| 8361 | * is only 1 task on the busy runqueue (because we don't call |
Kirill Tkhai | 163122b | 2014-08-20 13:48:29 +0400 | [diff] [blame] | 8362 | * detach_tasks). |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8363 | */ |
| 8364 | if (sd->balance_interval < sd->max_interval) |
| 8365 | sd->balance_interval *= 2; |
| 8366 | } |
| 8367 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8368 | goto out; |
| 8369 | |
| 8370 | out_balanced: |
Vincent Guittot | afdeee0 | 2014-08-26 13:06:44 +0200 | [diff] [blame] | 8371 | /* |
| 8372 | * We reach balance although we may have faced some affinity |
| 8373 | * constraints. Clear the imbalance flag if it was set. |
| 8374 | */ |
| 8375 | if (sd_parent) { |
| 8376 | int *group_imbalance = &sd_parent->groups->sgc->imbalance; |
| 8377 | |
| 8378 | if (*group_imbalance) |
| 8379 | *group_imbalance = 0; |
| 8380 | } |
| 8381 | |
| 8382 | out_all_pinned: |
| 8383 | /* |
| 8384 | * We reach balance because all tasks are pinned at this level so |
| 8385 | * we can't migrate them. Let the imbalance flag set so parent level |
| 8386 | * can try to migrate them. |
| 8387 | */ |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8388 | schedstat_inc(sd->lb_balanced[idle]); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8389 | |
| 8390 | sd->nr_balance_failed = 0; |
| 8391 | |
| 8392 | out_one_pinned: |
| 8393 | /* tune up the balancing interval */ |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8394 | if (((env.flags & LBF_ALL_PINNED) && |
Peter Zijlstra | 5b54b56 | 2011-09-22 15:23:13 +0200 | [diff] [blame] | 8395 | sd->balance_interval < MAX_PINNED_INTERVAL) || |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8396 | (sd->balance_interval < sd->max_interval)) |
| 8397 | sd->balance_interval *= 2; |
| 8398 | |
Venkatesh Pallipadi | 46e49b3 | 2011-02-14 14:38:50 -0800 | [diff] [blame] | 8399 | ld_moved = 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8400 | out: |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8401 | return ld_moved; |
| 8402 | } |
| 8403 | |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8404 | static inline unsigned long |
| 8405 | get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) |
| 8406 | { |
| 8407 | unsigned long interval = sd->balance_interval; |
| 8408 | |
| 8409 | if (cpu_busy) |
| 8410 | interval *= sd->busy_factor; |
| 8411 | |
| 8412 | /* scale ms to jiffies */ |
| 8413 | interval = msecs_to_jiffies(interval); |
| 8414 | interval = clamp(interval, 1UL, max_load_balance_interval); |
| 8415 | |
| 8416 | return interval; |
| 8417 | } |
| 8418 | |
| 8419 | static inline void |
Leo Yan | 31851a9 | 2016-08-05 14:31:29 +0800 | [diff] [blame] | 8420 | update_next_balance(struct sched_domain *sd, unsigned long *next_balance) |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8421 | { |
| 8422 | unsigned long interval, next; |
| 8423 | |
Leo Yan | 31851a9 | 2016-08-05 14:31:29 +0800 | [diff] [blame] | 8424 | /* used by idle balance, so cpu_busy = 0 */ |
| 8425 | interval = get_sd_balance_interval(sd, 0); |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8426 | next = sd->last_balance + interval; |
| 8427 | |
| 8428 | if (time_after(*next_balance, next)) |
| 8429 | *next_balance = next; |
| 8430 | } |
| 8431 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8432 | /* |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8433 | * idle_balance is called by schedule() if this_cpu is about to become |
| 8434 | * idle. Attempts to pull tasks from other CPUs. |
| 8435 | */ |
Matt Fleming | 46f69fa | 2016-09-21 14:38:12 +0100 | [diff] [blame] | 8436 | static int idle_balance(struct rq *this_rq, struct rq_flags *rf) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8437 | { |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8438 | unsigned long next_balance = jiffies + HZ; |
| 8439 | int this_cpu = this_rq->cpu; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8440 | struct sched_domain *sd; |
| 8441 | int pulled_task = 0; |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 8442 | u64 curr_cost = 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8443 | |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 8444 | /* |
| 8445 | * We must set idle_stamp _before_ calling idle_balance(), such that we |
| 8446 | * measure the duration of idle_balance() as idle time. |
| 8447 | */ |
| 8448 | this_rq->idle_stamp = rq_clock(this_rq); |
| 8449 | |
Matt Fleming | 46f69fa | 2016-09-21 14:38:12 +0100 | [diff] [blame] | 8450 | /* |
| 8451 | * This is OK, because current is on_cpu, which avoids it being picked |
| 8452 | * for load-balance and preemption/IRQs are still disabled avoiding |
| 8453 | * further scheduler activity on it and we're being very careful to |
| 8454 | * re-start the picking loop. |
| 8455 | */ |
| 8456 | rq_unpin_lock(this_rq, rf); |
| 8457 | |
Tim Chen | 4486edd | 2014-06-23 12:16:49 -0700 | [diff] [blame] | 8458 | if (this_rq->avg_idle < sysctl_sched_migration_cost || |
| 8459 | !this_rq->rd->overload) { |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8460 | rcu_read_lock(); |
| 8461 | sd = rcu_dereference_check_sched_domain(this_rq->sd); |
| 8462 | if (sd) |
Leo Yan | 31851a9 | 2016-08-05 14:31:29 +0800 | [diff] [blame] | 8463 | update_next_balance(sd, &next_balance); |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8464 | rcu_read_unlock(); |
| 8465 | |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 8466 | goto out; |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8467 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8468 | |
Peter Zijlstra | f492e12 | 2009-12-23 15:29:42 +0100 | [diff] [blame] | 8469 | raw_spin_unlock(&this_rq->lock); |
| 8470 | |
Paul Turner | 48a1675 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 8471 | update_blocked_averages(this_cpu); |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 8472 | rcu_read_lock(); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8473 | for_each_domain(this_cpu, sd) { |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8474 | int continue_balancing = 1; |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 8475 | u64 t0, domain_cost; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8476 | |
| 8477 | if (!(sd->flags & SD_LOAD_BALANCE)) |
| 8478 | continue; |
| 8479 | |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8480 | if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { |
Leo Yan | 31851a9 | 2016-08-05 14:31:29 +0800 | [diff] [blame] | 8481 | update_next_balance(sd, &next_balance); |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 8482 | break; |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8483 | } |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 8484 | |
Peter Zijlstra | f492e12 | 2009-12-23 15:29:42 +0100 | [diff] [blame] | 8485 | if (sd->flags & SD_BALANCE_NEWIDLE) { |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 8486 | t0 = sched_clock_cpu(this_cpu); |
| 8487 | |
Peter Zijlstra | f492e12 | 2009-12-23 15:29:42 +0100 | [diff] [blame] | 8488 | pulled_task = load_balance(this_cpu, this_rq, |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8489 | sd, CPU_NEWLY_IDLE, |
| 8490 | &continue_balancing); |
Jason Low | 9bd721c | 2013-09-13 11:26:52 -0700 | [diff] [blame] | 8491 | |
| 8492 | domain_cost = sched_clock_cpu(this_cpu) - t0; |
| 8493 | if (domain_cost > sd->max_newidle_lb_cost) |
| 8494 | sd->max_newidle_lb_cost = domain_cost; |
| 8495 | |
| 8496 | curr_cost += domain_cost; |
Peter Zijlstra | f492e12 | 2009-12-23 15:29:42 +0100 | [diff] [blame] | 8497 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8498 | |
Leo Yan | 31851a9 | 2016-08-05 14:31:29 +0800 | [diff] [blame] | 8499 | update_next_balance(sd, &next_balance); |
Jason Low | 39a4d9c | 2014-04-23 18:30:35 -0700 | [diff] [blame] | 8500 | |
| 8501 | /* |
| 8502 | * Stop searching for tasks to pull if there are |
| 8503 | * now runnable tasks on this rq. |
| 8504 | */ |
| 8505 | if (pulled_task || this_rq->nr_running > 0) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8506 | break; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8507 | } |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 8508 | rcu_read_unlock(); |
Peter Zijlstra | f492e12 | 2009-12-23 15:29:42 +0100 | [diff] [blame] | 8509 | |
| 8510 | raw_spin_lock(&this_rq->lock); |
| 8511 | |
Jason Low | 0e5b533 | 2014-04-28 15:45:54 -0700 | [diff] [blame] | 8512 | if (curr_cost > this_rq->max_idle_balance_cost) |
| 8513 | this_rq->max_idle_balance_cost = curr_cost; |
| 8514 | |
Daniel Lezcano | e5fc661 | 2014-01-17 10:04:02 +0100 | [diff] [blame] | 8515 | /* |
Jason Low | 0e5b533 | 2014-04-28 15:45:54 -0700 | [diff] [blame] | 8516 | * While browsing the domains, we released the rq lock, a task could |
| 8517 | * have been enqueued in the meantime. Since we're not going idle, |
| 8518 | * pretend we pulled a task. |
Daniel Lezcano | e5fc661 | 2014-01-17 10:04:02 +0100 | [diff] [blame] | 8519 | */ |
Jason Low | 0e5b533 | 2014-04-28 15:45:54 -0700 | [diff] [blame] | 8520 | if (this_rq->cfs.h_nr_running && !pulled_task) |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 8521 | pulled_task = 1; |
Daniel Lezcano | e5fc661 | 2014-01-17 10:04:02 +0100 | [diff] [blame] | 8522 | |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 8523 | out: |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8524 | /* Move the next balance forward */ |
| 8525 | if (time_after(this_rq->next_balance, next_balance)) |
| 8526 | this_rq->next_balance = next_balance; |
| 8527 | |
Kirill Tkhai | e4aa358 | 2014-03-06 13:31:55 +0400 | [diff] [blame] | 8528 | /* Is there a task of a high priority class? */ |
Kirill Tkhai | 4638364 | 2014-03-15 02:15:07 +0400 | [diff] [blame] | 8529 | if (this_rq->nr_running != this_rq->cfs.h_nr_running) |
Kirill Tkhai | e4aa358 | 2014-03-06 13:31:55 +0400 | [diff] [blame] | 8530 | pulled_task = -1; |
| 8531 | |
Dietmar Eggemann | 38c6ade | 2015-10-20 13:04:41 +0100 | [diff] [blame] | 8532 | if (pulled_task) |
Peter Zijlstra | 6e83125 | 2014-02-11 16:11:48 +0100 | [diff] [blame] | 8533 | this_rq->idle_stamp = 0; |
| 8534 | |
Matt Fleming | 46f69fa | 2016-09-21 14:38:12 +0100 | [diff] [blame] | 8535 | rq_repin_lock(this_rq, rf); |
| 8536 | |
Daniel Lezcano | 3c4017c | 2014-01-17 10:04:03 +0100 | [diff] [blame] | 8537 | return pulled_task; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8538 | } |
| 8539 | |
| 8540 | /* |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8541 | * active_load_balance_cpu_stop is run by cpu stopper. It pushes |
| 8542 | * running tasks off the busiest CPU onto idle CPUs. It requires at |
| 8543 | * least 1 task to be running on each physical CPU where possible, and |
| 8544 | * avoids physical / logical imbalances. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8545 | */ |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8546 | static int active_load_balance_cpu_stop(void *data) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8547 | { |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8548 | struct rq *busiest_rq = data; |
| 8549 | int busiest_cpu = cpu_of(busiest_rq); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8550 | int target_cpu = busiest_rq->push_cpu; |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8551 | struct rq *target_rq = cpu_rq(target_cpu); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8552 | struct sched_domain *sd; |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 8553 | struct task_struct *p = NULL; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8554 | struct rq_flags rf; |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8555 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8556 | rq_lock_irq(busiest_rq, &rf); |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8557 | |
| 8558 | /* make sure the requested cpu hasn't gone down in the meantime */ |
| 8559 | if (unlikely(busiest_cpu != smp_processor_id() || |
| 8560 | !busiest_rq->active_balance)) |
| 8561 | goto out_unlock; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8562 | |
| 8563 | /* Is there any task to move? */ |
| 8564 | if (busiest_rq->nr_running <= 1) |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8565 | goto out_unlock; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8566 | |
| 8567 | /* |
| 8568 | * This condition is "impossible", if it occurs |
| 8569 | * we need to fix it. Originally reported by |
| 8570 | * Bjorn Helgaas on a 128-cpu setup. |
| 8571 | */ |
| 8572 | BUG_ON(busiest_rq == target_rq); |
| 8573 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8574 | /* Search for an sd spanning us and the target CPU. */ |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 8575 | rcu_read_lock(); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8576 | for_each_domain(target_cpu, sd) { |
| 8577 | if ((sd->flags & SD_LOAD_BALANCE) && |
| 8578 | cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) |
| 8579 | break; |
| 8580 | } |
| 8581 | |
| 8582 | if (likely(sd)) { |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8583 | struct lb_env env = { |
| 8584 | .sd = sd, |
Peter Zijlstra | ddcdf6e | 2012-02-22 19:27:40 +0100 | [diff] [blame] | 8585 | .dst_cpu = target_cpu, |
| 8586 | .dst_rq = target_rq, |
| 8587 | .src_cpu = busiest_rq->cpu, |
| 8588 | .src_rq = busiest_rq, |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8589 | .idle = CPU_IDLE, |
Jeffrey Hugo | 65a4433 | 2017-06-07 13:18:57 -0600 | [diff] [blame] | 8590 | /* |
| 8591 | * can_migrate_task() doesn't need to compute new_dst_cpu |
| 8592 | * for active balancing. Since we have CPU_IDLE, but no |
| 8593 | * @dst_grpmask we need to make that test go away with lying |
| 8594 | * about DST_PINNED. |
| 8595 | */ |
| 8596 | .flags = LBF_DST_PINNED, |
Peter Zijlstra | 8e45cb5 | 2012-02-22 12:47:19 +0100 | [diff] [blame] | 8597 | }; |
| 8598 | |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8599 | schedstat_inc(sd->alb_count); |
Peter Zijlstra | 3bed5e2 | 2016-10-03 16:35:32 +0200 | [diff] [blame] | 8600 | update_rq_clock(busiest_rq); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8601 | |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 8602 | p = detach_one_task(&env); |
Srikar Dronamraju | d02c07118 | 2016-03-23 17:54:44 +0530 | [diff] [blame] | 8603 | if (p) { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8604 | schedstat_inc(sd->alb_pushed); |
Srikar Dronamraju | d02c07118 | 2016-03-23 17:54:44 +0530 | [diff] [blame] | 8605 | /* Active balancing done, reset the failure counter. */ |
| 8606 | sd->nr_balance_failed = 0; |
| 8607 | } else { |
Josh Poimboeuf | ae92882 | 2016-06-17 12:43:24 -0500 | [diff] [blame] | 8608 | schedstat_inc(sd->alb_failed); |
Srikar Dronamraju | d02c07118 | 2016-03-23 17:54:44 +0530 | [diff] [blame] | 8609 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8610 | } |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 8611 | rcu_read_unlock(); |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8612 | out_unlock: |
| 8613 | busiest_rq->active_balance = 0; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8614 | rq_unlock(busiest_rq, &rf); |
Kirill Tkhai | e5673f2 | 2014-08-20 13:48:01 +0400 | [diff] [blame] | 8615 | |
| 8616 | if (p) |
| 8617 | attach_one_task(target_rq, p); |
| 8618 | |
| 8619 | local_irq_enable(); |
| 8620 | |
Tejun Heo | 969c792 | 2010-05-06 18:49:21 +0200 | [diff] [blame] | 8621 | return 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8622 | } |
| 8623 | |
Mike Galbraith | d987fc7 | 2011-12-05 10:01:47 +0100 | [diff] [blame] | 8624 | static inline int on_null_domain(struct rq *rq) |
| 8625 | { |
| 8626 | return unlikely(!rcu_dereference_sched(rq->sd)); |
| 8627 | } |
| 8628 | |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 8629 | #ifdef CONFIG_NO_HZ_COMMON |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8630 | /* |
| 8631 | * idle load balancing details |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8632 | * - When one of the busy CPUs notice that there may be an idle rebalancing |
| 8633 | * needed, they will kick the idle load balancer, which then does idle |
| 8634 | * load balancing for all the idle CPUs. |
| 8635 | */ |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8636 | static struct { |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8637 | cpumask_var_t idle_cpus_mask; |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8638 | atomic_t nr_cpus; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8639 | unsigned long next_balance; /* in jiffy units */ |
| 8640 | } nohz ____cacheline_aligned; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8641 | |
Daniel Lezcano | 3dd0337 | 2014-01-06 12:34:41 +0100 | [diff] [blame] | 8642 | static inline int find_new_ilb(void) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8643 | { |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8644 | int ilb = cpumask_first(nohz.idle_cpus_mask); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8645 | |
Suresh Siddha | 786d6dc | 2011-12-01 17:07:35 -0800 | [diff] [blame] | 8646 | if (ilb < nr_cpu_ids && idle_cpu(ilb)) |
| 8647 | return ilb; |
| 8648 | |
| 8649 | return nr_cpu_ids; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8650 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8651 | |
| 8652 | /* |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8653 | * Kick a CPU to do the nohz balancing, if it is time for it. We pick the |
| 8654 | * nohz_load_balancer CPU (if there is one) otherwise fallback to any idle |
| 8655 | * CPU (if there is one). |
| 8656 | */ |
Daniel Lezcano | 0aeeeeb | 2014-01-06 12:34:42 +0100 | [diff] [blame] | 8657 | static void nohz_balancer_kick(void) |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8658 | { |
| 8659 | int ilb_cpu; |
| 8660 | |
| 8661 | nohz.next_balance++; |
| 8662 | |
Daniel Lezcano | 3dd0337 | 2014-01-06 12:34:41 +0100 | [diff] [blame] | 8663 | ilb_cpu = find_new_ilb(); |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8664 | |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8665 | if (ilb_cpu >= nr_cpu_ids) |
| 8666 | return; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8667 | |
Suresh Siddha | cd490c5 | 2011-12-06 11:26:34 -0800 | [diff] [blame] | 8668 | if (test_and_set_bit(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu))) |
Suresh Siddha | 1c792db | 2011-12-01 17:07:32 -0800 | [diff] [blame] | 8669 | return; |
| 8670 | /* |
| 8671 | * Use smp_send_reschedule() instead of resched_cpu(). |
| 8672 | * This way we generate a sched IPI on the target cpu which |
| 8673 | * is idle. And the softirq performing nohz idle load balance |
| 8674 | * will be run before returning from the IPI. |
| 8675 | */ |
| 8676 | smp_send_reschedule(ilb_cpu); |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8677 | return; |
| 8678 | } |
| 8679 | |
Thomas Gleixner | 20a5c8c | 2016-03-10 12:54:20 +0100 | [diff] [blame] | 8680 | void nohz_balance_exit_idle(unsigned int cpu) |
Suresh Siddha | 7132596 | 2012-01-19 18:28:57 -0800 | [diff] [blame] | 8681 | { |
| 8682 | if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) { |
Mike Galbraith | d987fc7 | 2011-12-05 10:01:47 +0100 | [diff] [blame] | 8683 | /* |
| 8684 | * Completely isolated CPUs don't ever set, so we must test. |
| 8685 | */ |
| 8686 | if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) { |
| 8687 | cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); |
| 8688 | atomic_dec(&nohz.nr_cpus); |
| 8689 | } |
Suresh Siddha | 7132596 | 2012-01-19 18:28:57 -0800 | [diff] [blame] | 8690 | clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); |
| 8691 | } |
| 8692 | } |
| 8693 | |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8694 | static inline void set_cpu_sd_state_busy(void) |
| 8695 | { |
| 8696 | struct sched_domain *sd; |
Preeti U Murthy | 37dc6b5 | 2013-10-30 08:42:52 +0530 | [diff] [blame] | 8697 | int cpu = smp_processor_id(); |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8698 | |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8699 | rcu_read_lock(); |
Peter Zijlstra | 0e369d7 | 2016-05-09 10:38:01 +0200 | [diff] [blame] | 8700 | sd = rcu_dereference(per_cpu(sd_llc, cpu)); |
Vincent Guittot | 25f55d9 | 2013-04-23 16:59:02 +0200 | [diff] [blame] | 8701 | |
| 8702 | if (!sd || !sd->nohz_idle) |
| 8703 | goto unlock; |
| 8704 | sd->nohz_idle = 0; |
| 8705 | |
Peter Zijlstra | 0e369d7 | 2016-05-09 10:38:01 +0200 | [diff] [blame] | 8706 | atomic_inc(&sd->shared->nr_busy_cpus); |
Vincent Guittot | 25f55d9 | 2013-04-23 16:59:02 +0200 | [diff] [blame] | 8707 | unlock: |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8708 | rcu_read_unlock(); |
| 8709 | } |
| 8710 | |
| 8711 | void set_cpu_sd_state_idle(void) |
| 8712 | { |
| 8713 | struct sched_domain *sd; |
Preeti U Murthy | 37dc6b5 | 2013-10-30 08:42:52 +0530 | [diff] [blame] | 8714 | int cpu = smp_processor_id(); |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8715 | |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8716 | rcu_read_lock(); |
Peter Zijlstra | 0e369d7 | 2016-05-09 10:38:01 +0200 | [diff] [blame] | 8717 | sd = rcu_dereference(per_cpu(sd_llc, cpu)); |
Vincent Guittot | 25f55d9 | 2013-04-23 16:59:02 +0200 | [diff] [blame] | 8718 | |
| 8719 | if (!sd || sd->nohz_idle) |
| 8720 | goto unlock; |
| 8721 | sd->nohz_idle = 1; |
| 8722 | |
Peter Zijlstra | 0e369d7 | 2016-05-09 10:38:01 +0200 | [diff] [blame] | 8723 | atomic_dec(&sd->shared->nr_busy_cpus); |
Vincent Guittot | 25f55d9 | 2013-04-23 16:59:02 +0200 | [diff] [blame] | 8724 | unlock: |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8725 | rcu_read_unlock(); |
| 8726 | } |
| 8727 | |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8728 | /* |
Alex Shi | c1cc017 | 2012-09-10 15:10:58 +0800 | [diff] [blame] | 8729 | * This routine will record that the cpu is going idle with tick stopped. |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8730 | * This info will be used in performing idle load balancing in the future. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8731 | */ |
Alex Shi | c1cc017 | 2012-09-10 15:10:58 +0800 | [diff] [blame] | 8732 | void nohz_balance_enter_idle(int cpu) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8733 | { |
Suresh Siddha | 7132596 | 2012-01-19 18:28:57 -0800 | [diff] [blame] | 8734 | /* |
| 8735 | * If this cpu is going down, then nothing needs to be done. |
| 8736 | */ |
| 8737 | if (!cpu_active(cpu)) |
| 8738 | return; |
| 8739 | |
Frederic Weisbecker | 387bc8b | 2017-06-19 04:12:02 +0200 | [diff] [blame] | 8740 | /* Spare idle load balancing on CPUs that don't want to be disturbed: */ |
| 8741 | if (!is_housekeeping_cpu(cpu)) |
| 8742 | return; |
| 8743 | |
Alex Shi | c1cc017 | 2012-09-10 15:10:58 +0800 | [diff] [blame] | 8744 | if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) |
| 8745 | return; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8746 | |
Mike Galbraith | d987fc7 | 2011-12-05 10:01:47 +0100 | [diff] [blame] | 8747 | /* |
| 8748 | * If we're a completely isolated CPU, we don't play. |
| 8749 | */ |
| 8750 | if (on_null_domain(cpu_rq(cpu))) |
| 8751 | return; |
| 8752 | |
Alex Shi | c1cc017 | 2012-09-10 15:10:58 +0800 | [diff] [blame] | 8753 | cpumask_set_cpu(cpu, nohz.idle_cpus_mask); |
| 8754 | atomic_inc(&nohz.nr_cpus); |
| 8755 | set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8756 | } |
| 8757 | #endif |
| 8758 | |
| 8759 | static DEFINE_SPINLOCK(balancing); |
| 8760 | |
Peter Zijlstra | 49c022e | 2011-04-05 10:14:25 +0200 | [diff] [blame] | 8761 | /* |
| 8762 | * Scale the max load_balance interval with the number of CPUs in the system. |
| 8763 | * This trades load-balance latency on larger machines for less cross talk. |
| 8764 | */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 8765 | void update_max_interval(void) |
Peter Zijlstra | 49c022e | 2011-04-05 10:14:25 +0200 | [diff] [blame] | 8766 | { |
| 8767 | max_load_balance_interval = HZ*num_online_cpus()/10; |
| 8768 | } |
| 8769 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8770 | /* |
| 8771 | * It checks each scheduling domain to see if it is due to be balanced, |
| 8772 | * and initiates a balancing operation if so. |
| 8773 | * |
Libin | b9b0853 | 2013-04-01 19:14:01 +0800 | [diff] [blame] | 8774 | * Balancing parameters are set up in init_sched_domains. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8775 | */ |
Daniel Lezcano | f7ed0a8 | 2014-01-06 12:34:43 +0100 | [diff] [blame] | 8776 | static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8777 | { |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8778 | int continue_balancing = 1; |
Daniel Lezcano | f7ed0a8 | 2014-01-06 12:34:43 +0100 | [diff] [blame] | 8779 | int cpu = rq->cpu; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8780 | unsigned long interval; |
Peter Zijlstra | 04f733b | 2012-05-11 00:12:02 +0200 | [diff] [blame] | 8781 | struct sched_domain *sd; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8782 | /* Earliest time when we have to do rebalance again */ |
| 8783 | unsigned long next_balance = jiffies + 60*HZ; |
| 8784 | int update_next_balance = 0; |
Jason Low | f48627e | 2013-09-13 11:26:53 -0700 | [diff] [blame] | 8785 | int need_serialize, need_decay = 0; |
| 8786 | u64 max_cost = 0; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8787 | |
Paul Turner | 48a1675 | 2012-10-04 13:18:31 +0200 | [diff] [blame] | 8788 | update_blocked_averages(cpu); |
Peter Zijlstra | 2069dd7 | 2010-11-15 15:47:00 -0800 | [diff] [blame] | 8789 | |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 8790 | rcu_read_lock(); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8791 | for_each_domain(cpu, sd) { |
Jason Low | f48627e | 2013-09-13 11:26:53 -0700 | [diff] [blame] | 8792 | /* |
| 8793 | * Decay the newidle max times here because this is a regular |
| 8794 | * visit to all the domains. Decay ~1% per second. |
| 8795 | */ |
| 8796 | if (time_after(jiffies, sd->next_decay_max_lb_cost)) { |
| 8797 | sd->max_newidle_lb_cost = |
| 8798 | (sd->max_newidle_lb_cost * 253) / 256; |
| 8799 | sd->next_decay_max_lb_cost = jiffies + HZ; |
| 8800 | need_decay = 1; |
| 8801 | } |
| 8802 | max_cost += sd->max_newidle_lb_cost; |
| 8803 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8804 | if (!(sd->flags & SD_LOAD_BALANCE)) |
| 8805 | continue; |
| 8806 | |
Jason Low | f48627e | 2013-09-13 11:26:53 -0700 | [diff] [blame] | 8807 | /* |
| 8808 | * Stop the load balance at this level. There is another |
| 8809 | * CPU in our sched group which is doing load balancing more |
| 8810 | * actively. |
| 8811 | */ |
| 8812 | if (!continue_balancing) { |
| 8813 | if (need_decay) |
| 8814 | continue; |
| 8815 | break; |
| 8816 | } |
| 8817 | |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8818 | interval = get_sd_balance_interval(sd, idle != CPU_IDLE); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8819 | |
| 8820 | need_serialize = sd->flags & SD_SERIALIZE; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8821 | if (need_serialize) { |
| 8822 | if (!spin_trylock(&balancing)) |
| 8823 | goto out; |
| 8824 | } |
| 8825 | |
| 8826 | if (time_after_eq(jiffies, sd->last_balance + interval)) { |
Joonsoo Kim | 23f0d20 | 2013-08-06 17:36:42 +0900 | [diff] [blame] | 8827 | if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8828 | /* |
Peter Zijlstra | 6263322 | 2013-08-19 12:41:09 +0200 | [diff] [blame] | 8829 | * The LBF_DST_PINNED logic could have changed |
Joonsoo Kim | de5eb2d | 2013-04-23 17:27:38 +0900 | [diff] [blame] | 8830 | * env->dst_cpu, so we can't know our idle |
| 8831 | * state even if we migrated tasks. Update it. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8832 | */ |
Joonsoo Kim | de5eb2d | 2013-04-23 17:27:38 +0900 | [diff] [blame] | 8833 | idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8834 | } |
| 8835 | sd->last_balance = jiffies; |
Jason Low | 52a08ef | 2014-05-08 17:49:22 -0700 | [diff] [blame] | 8836 | interval = get_sd_balance_interval(sd, idle != CPU_IDLE); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8837 | } |
| 8838 | if (need_serialize) |
| 8839 | spin_unlock(&balancing); |
| 8840 | out: |
| 8841 | if (time_after(next_balance, sd->last_balance + interval)) { |
| 8842 | next_balance = sd->last_balance + interval; |
| 8843 | update_next_balance = 1; |
| 8844 | } |
Jason Low | f48627e | 2013-09-13 11:26:53 -0700 | [diff] [blame] | 8845 | } |
| 8846 | if (need_decay) { |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8847 | /* |
Jason Low | f48627e | 2013-09-13 11:26:53 -0700 | [diff] [blame] | 8848 | * Ensure the rq-wide value also decays but keep it at a |
| 8849 | * reasonable floor to avoid funnies with rq->avg_idle. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8850 | */ |
Jason Low | f48627e | 2013-09-13 11:26:53 -0700 | [diff] [blame] | 8851 | rq->max_idle_balance_cost = |
| 8852 | max((u64)sysctl_sched_migration_cost, max_cost); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8853 | } |
Peter Zijlstra | dce840a | 2011-04-07 14:09:50 +0200 | [diff] [blame] | 8854 | rcu_read_unlock(); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8855 | |
| 8856 | /* |
| 8857 | * next_balance will be updated only when there is a need. |
| 8858 | * When the cpu is attached to null domain for ex, it will not be |
| 8859 | * updated. |
| 8860 | */ |
Vincent Guittot | c5afb6a | 2015-08-03 11:55:50 +0200 | [diff] [blame] | 8861 | if (likely(update_next_balance)) { |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8862 | rq->next_balance = next_balance; |
Vincent Guittot | c5afb6a | 2015-08-03 11:55:50 +0200 | [diff] [blame] | 8863 | |
| 8864 | #ifdef CONFIG_NO_HZ_COMMON |
| 8865 | /* |
| 8866 | * If this CPU has been elected to perform the nohz idle |
| 8867 | * balance. Other idle CPUs have already rebalanced with |
| 8868 | * nohz_idle_balance() and nohz.next_balance has been |
| 8869 | * updated accordingly. This CPU is now running the idle load |
| 8870 | * balance for itself and we need to update the |
| 8871 | * nohz.next_balance accordingly. |
| 8872 | */ |
| 8873 | if ((idle == CPU_IDLE) && time_after(nohz.next_balance, rq->next_balance)) |
| 8874 | nohz.next_balance = rq->next_balance; |
| 8875 | #endif |
| 8876 | } |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 8877 | } |
| 8878 | |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 8879 | #ifdef CONFIG_NO_HZ_COMMON |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8880 | /* |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 8881 | * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8882 | * rebalancing for all the cpus for whom scheduler ticks are stopped. |
| 8883 | */ |
Daniel Lezcano | 208cb16 | 2014-01-06 12:34:44 +0100 | [diff] [blame] | 8884 | static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8885 | { |
Daniel Lezcano | 208cb16 | 2014-01-06 12:34:44 +0100 | [diff] [blame] | 8886 | int this_cpu = this_rq->cpu; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8887 | struct rq *rq; |
| 8888 | int balance_cpu; |
Vincent Guittot | c5afb6a | 2015-08-03 11:55:50 +0200 | [diff] [blame] | 8889 | /* Earliest time when we have to do rebalance again */ |
| 8890 | unsigned long next_balance = jiffies + 60*HZ; |
| 8891 | int update_next_balance = 0; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8892 | |
Suresh Siddha | 1c792db | 2011-12-01 17:07:32 -0800 | [diff] [blame] | 8893 | if (idle != CPU_IDLE || |
| 8894 | !test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu))) |
| 8895 | goto end; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8896 | |
| 8897 | for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { |
Suresh Siddha | 8a6d42d | 2011-12-06 11:19:37 -0800 | [diff] [blame] | 8898 | if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8899 | continue; |
| 8900 | |
| 8901 | /* |
| 8902 | * If this cpu gets work to do, stop the load balancing |
| 8903 | * work being done for other cpus. Next load |
| 8904 | * balancing owner will pick it up. |
| 8905 | */ |
Suresh Siddha | 1c792db | 2011-12-01 17:07:32 -0800 | [diff] [blame] | 8906 | if (need_resched()) |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8907 | break; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8908 | |
Vincent Guittot | 5ed4f1d | 2012-09-13 06:11:26 +0200 | [diff] [blame] | 8909 | rq = cpu_rq(balance_cpu); |
| 8910 | |
Tim Chen | ed61bbc | 2014-05-20 14:39:27 -0700 | [diff] [blame] | 8911 | /* |
| 8912 | * If time for next balance is due, |
| 8913 | * do the balance. |
| 8914 | */ |
| 8915 | if (time_after_eq(jiffies, rq->next_balance)) { |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8916 | struct rq_flags rf; |
| 8917 | |
| 8918 | rq_lock_irq(rq, &rf); |
Tim Chen | ed61bbc | 2014-05-20 14:39:27 -0700 | [diff] [blame] | 8919 | update_rq_clock(rq); |
Frederic Weisbecker | cee1afc | 2016-04-13 15:56:50 +0200 | [diff] [blame] | 8920 | cpu_load_update_idle(rq); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 8921 | rq_unlock_irq(rq, &rf); |
| 8922 | |
Tim Chen | ed61bbc | 2014-05-20 14:39:27 -0700 | [diff] [blame] | 8923 | rebalance_domains(rq, CPU_IDLE); |
| 8924 | } |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8925 | |
Vincent Guittot | c5afb6a | 2015-08-03 11:55:50 +0200 | [diff] [blame] | 8926 | if (time_after(next_balance, rq->next_balance)) { |
| 8927 | next_balance = rq->next_balance; |
| 8928 | update_next_balance = 1; |
| 8929 | } |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8930 | } |
Vincent Guittot | c5afb6a | 2015-08-03 11:55:50 +0200 | [diff] [blame] | 8931 | |
| 8932 | /* |
| 8933 | * next_balance will be updated only when there is a need. |
| 8934 | * When the CPU is attached to null domain for ex, it will not be |
| 8935 | * updated. |
| 8936 | */ |
| 8937 | if (likely(update_next_balance)) |
| 8938 | nohz.next_balance = next_balance; |
Suresh Siddha | 1c792db | 2011-12-01 17:07:32 -0800 | [diff] [blame] | 8939 | end: |
| 8940 | clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8941 | } |
| 8942 | |
| 8943 | /* |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8944 | * Current heuristic for kicking the idle load balancer in the presence |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8945 | * of an idle cpu in the system. |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8946 | * - This rq has more than one task. |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8947 | * - This rq has at least one CFS task and the capacity of the CPU is |
| 8948 | * significantly reduced because of RT tasks or IRQs. |
| 8949 | * - At parent of LLC scheduler domain level, this cpu's scheduler group has |
| 8950 | * multiple busy cpu. |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8951 | * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler |
| 8952 | * domain span are idle. |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8953 | */ |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8954 | static inline bool nohz_kick_needed(struct rq *rq) |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8955 | { |
| 8956 | unsigned long now = jiffies; |
Peter Zijlstra | 0e369d7 | 2016-05-09 10:38:01 +0200 | [diff] [blame] | 8957 | struct sched_domain_shared *sds; |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8958 | struct sched_domain *sd; |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 8959 | int nr_busy, i, cpu = rq->cpu; |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8960 | bool kick = false; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8961 | |
Daniel Lezcano | 4a72562 | 2014-01-06 12:34:39 +0100 | [diff] [blame] | 8962 | if (unlikely(rq->idle_balance)) |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8963 | return false; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8964 | |
Suresh Siddha | 1c792db | 2011-12-01 17:07:32 -0800 | [diff] [blame] | 8965 | /* |
| 8966 | * We may be recently in ticked or tickless idle mode. At the first |
| 8967 | * busy tick after returning from idle, we will update the busy stats. |
| 8968 | */ |
Suresh Siddha | 69e1e81 | 2011-12-01 17:07:33 -0800 | [diff] [blame] | 8969 | set_cpu_sd_state_busy(); |
Alex Shi | c1cc017 | 2012-09-10 15:10:58 +0800 | [diff] [blame] | 8970 | nohz_balance_exit_idle(cpu); |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8971 | |
| 8972 | /* |
| 8973 | * None are in tickless mode and hence no need for NOHZ idle load |
| 8974 | * balancing. |
| 8975 | */ |
| 8976 | if (likely(!atomic_read(&nohz.nr_cpus))) |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8977 | return false; |
Suresh Siddha | 1c792db | 2011-12-01 17:07:32 -0800 | [diff] [blame] | 8978 | |
| 8979 | if (time_before(now, nohz.next_balance)) |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8980 | return false; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8981 | |
Suresh Siddha | 0b005cf | 2011-12-01 17:07:34 -0800 | [diff] [blame] | 8982 | if (rq->nr_running >= 2) |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8983 | return true; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 8984 | |
Peter Zijlstra | 067491b | 2011-12-07 14:32:08 +0100 | [diff] [blame] | 8985 | rcu_read_lock(); |
Peter Zijlstra | 0e369d7 | 2016-05-09 10:38:01 +0200 | [diff] [blame] | 8986 | sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); |
| 8987 | if (sds) { |
| 8988 | /* |
| 8989 | * XXX: write a coherent comment on why we do this. |
| 8990 | * See also: http://lkml.kernel.org/r/20111202010832.602203411@sbsiddha-desk.sc.intel.com |
| 8991 | */ |
| 8992 | nr_busy = atomic_read(&sds->nr_busy_cpus); |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 8993 | if (nr_busy > 1) { |
| 8994 | kick = true; |
| 8995 | goto unlock; |
| 8996 | } |
| 8997 | |
| 8998 | } |
| 8999 | |
| 9000 | sd = rcu_dereference(rq->sd); |
| 9001 | if (sd) { |
| 9002 | if ((rq->cfs.h_nr_running >= 1) && |
| 9003 | check_cpu_capacity(rq, sd)) { |
| 9004 | kick = true; |
| 9005 | goto unlock; |
| 9006 | } |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 9007 | } |
Preeti U Murthy | 37dc6b5 | 2013-10-30 08:42:52 +0530 | [diff] [blame] | 9008 | |
| 9009 | sd = rcu_dereference(per_cpu(sd_asym, cpu)); |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 9010 | if (sd) { |
| 9011 | for_each_cpu(i, sched_domain_span(sd)) { |
| 9012 | if (i == cpu || |
| 9013 | !cpumask_test_cpu(i, nohz.idle_cpus_mask)) |
| 9014 | continue; |
Preeti U Murthy | 37dc6b5 | 2013-10-30 08:42:52 +0530 | [diff] [blame] | 9015 | |
Tim Chen | afe06ef | 2016-11-22 12:23:53 -0800 | [diff] [blame] | 9016 | if (sched_asym_prefer(i, cpu)) { |
| 9017 | kick = true; |
| 9018 | goto unlock; |
| 9019 | } |
| 9020 | } |
| 9021 | } |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 9022 | unlock: |
Peter Zijlstra | 067491b | 2011-12-07 14:32:08 +0100 | [diff] [blame] | 9023 | rcu_read_unlock(); |
Vincent Guittot | 1aaf90a | 2015-02-27 16:54:14 +0100 | [diff] [blame] | 9024 | return kick; |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 9025 | } |
| 9026 | #else |
Daniel Lezcano | 208cb16 | 2014-01-06 12:34:44 +0100 | [diff] [blame] | 9027 | static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { } |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 9028 | #endif |
| 9029 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9030 | /* |
| 9031 | * run_rebalance_domains is triggered when needed from the scheduler tick. |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 9032 | * Also triggered for nohz idle balancing (with nohz_balancing_kick set). |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9033 | */ |
Emese Revfy | 0766f78 | 2016-06-20 20:42:34 +0200 | [diff] [blame] | 9034 | static __latent_entropy void run_rebalance_domains(struct softirq_action *h) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9035 | { |
Daniel Lezcano | 208cb16 | 2014-01-06 12:34:44 +0100 | [diff] [blame] | 9036 | struct rq *this_rq = this_rq(); |
Suresh Siddha | 6eb57e0 | 2011-10-03 15:09:01 -0700 | [diff] [blame] | 9037 | enum cpu_idle_type idle = this_rq->idle_balance ? |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9038 | CPU_IDLE : CPU_NOT_IDLE; |
| 9039 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9040 | /* |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 9041 | * If this cpu has a pending nohz_balance_kick, then do the |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9042 | * balancing on behalf of the other idle cpus whose ticks are |
Preeti U Murthy | d4573c3 | 2015-03-26 18:32:44 +0530 | [diff] [blame] | 9043 | * stopped. Do nohz_idle_balance *before* rebalance_domains to |
| 9044 | * give the idle cpus a chance to load balance. Else we may |
| 9045 | * load balance only within the local sched_domain hierarchy |
| 9046 | * and abort nohz_idle_balance altogether if we pull some load. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9047 | */ |
Daniel Lezcano | 208cb16 | 2014-01-06 12:34:44 +0100 | [diff] [blame] | 9048 | nohz_idle_balance(this_rq, idle); |
Preeti U Murthy | d4573c3 | 2015-03-26 18:32:44 +0530 | [diff] [blame] | 9049 | rebalance_domains(this_rq, idle); |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9050 | } |
| 9051 | |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9052 | /* |
| 9053 | * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9054 | */ |
Daniel Lezcano | 7caff66 | 2014-01-06 12:34:38 +0100 | [diff] [blame] | 9055 | void trigger_load_balance(struct rq *rq) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9056 | { |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9057 | /* Don't need to rebalance while attached to NULL domain */ |
Daniel Lezcano | c726099 | 2014-01-06 12:34:45 +0100 | [diff] [blame] | 9058 | if (unlikely(on_null_domain(rq))) |
| 9059 | return; |
| 9060 | |
| 9061 | if (time_after_eq(jiffies, rq->next_balance)) |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9062 | raise_softirq(SCHED_SOFTIRQ); |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 9063 | #ifdef CONFIG_NO_HZ_COMMON |
Daniel Lezcano | c726099 | 2014-01-06 12:34:45 +0100 | [diff] [blame] | 9064 | if (nohz_kick_needed(rq)) |
Daniel Lezcano | 0aeeeeb | 2014-01-06 12:34:42 +0100 | [diff] [blame] | 9065 | nohz_balancer_kick(); |
Venkatesh Pallipadi | 83cd4fe | 2010-05-21 17:09:41 -0700 | [diff] [blame] | 9066 | #endif |
Peter Zijlstra | 1e3c88b | 2009-12-17 17:00:43 +0100 | [diff] [blame] | 9067 | } |
| 9068 | |
Christian Ehrhardt | 0bcdcf2 | 2009-11-30 12:16:46 +0100 | [diff] [blame] | 9069 | static void rq_online_fair(struct rq *rq) |
| 9070 | { |
| 9071 | update_sysctl(); |
Kirill Tkhai | 0e59bda | 2014-06-25 12:19:42 +0400 | [diff] [blame] | 9072 | |
| 9073 | update_runtime_enabled(rq); |
Christian Ehrhardt | 0bcdcf2 | 2009-11-30 12:16:46 +0100 | [diff] [blame] | 9074 | } |
| 9075 | |
| 9076 | static void rq_offline_fair(struct rq *rq) |
| 9077 | { |
| 9078 | update_sysctl(); |
Peter Boonstoppel | a4c96ae | 2012-08-09 15:34:47 -0700 | [diff] [blame] | 9079 | |
| 9080 | /* Ensure any throttled groups are reachable by pick_next_task */ |
| 9081 | unthrottle_offline_cfs_rqs(rq); |
Christian Ehrhardt | 0bcdcf2 | 2009-11-30 12:16:46 +0100 | [diff] [blame] | 9082 | } |
| 9083 | |
Dhaval Giani | 55e12e5 | 2008-06-24 23:39:43 +0530 | [diff] [blame] | 9084 | #endif /* CONFIG_SMP */ |
Peter Williams | e1d1484 | 2007-10-24 18:23:51 +0200 | [diff] [blame] | 9085 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9086 | /* |
| 9087 | * scheduler tick hitting a task of our scheduling class: |
| 9088 | */ |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 9089 | static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9090 | { |
| 9091 | struct cfs_rq *cfs_rq; |
| 9092 | struct sched_entity *se = &curr->se; |
| 9093 | |
| 9094 | for_each_sched_entity(se) { |
| 9095 | cfs_rq = cfs_rq_of(se); |
Peter Zijlstra | 8f4d37e | 2008-01-25 21:08:29 +0100 | [diff] [blame] | 9096 | entity_tick(cfs_rq, se, queued); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9097 | } |
Ben Segall | 18bf280 | 2012-10-04 12:51:20 +0200 | [diff] [blame] | 9098 | |
Srikar Dronamraju | b52da86 | 2015-10-02 07:48:25 +0530 | [diff] [blame] | 9099 | if (static_branch_unlikely(&sched_numa_balancing)) |
Peter Zijlstra | cbee9f8 | 2012-10-25 14:16:43 +0200 | [diff] [blame] | 9100 | task_tick_numa(rq, curr); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9101 | } |
| 9102 | |
| 9103 | /* |
Peter Zijlstra | cd29fe6 | 2009-11-27 17:32:46 +0100 | [diff] [blame] | 9104 | * called on fork with the child task as argument from the parent's context |
| 9105 | * - child not yet on the tasklist |
| 9106 | * - preemption disabled |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9107 | */ |
Peter Zijlstra | cd29fe6 | 2009-11-27 17:32:46 +0100 | [diff] [blame] | 9108 | static void task_fork_fair(struct task_struct *p) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9109 | { |
Daisuke Nishimura | 4fc420c | 2011-12-15 14:36:55 +0900 | [diff] [blame] | 9110 | struct cfs_rq *cfs_rq; |
| 9111 | struct sched_entity *se = &p->se, *curr; |
Peter Zijlstra | cd29fe6 | 2009-11-27 17:32:46 +0100 | [diff] [blame] | 9112 | struct rq *rq = this_rq(); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9113 | struct rq_flags rf; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9114 | |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9115 | rq_lock(rq, &rf); |
Peter Zijlstra | 861d034 | 2010-08-19 13:31:43 +0200 | [diff] [blame] | 9116 | update_rq_clock(rq); |
| 9117 | |
Daisuke Nishimura | 4fc420c | 2011-12-15 14:36:55 +0900 | [diff] [blame] | 9118 | cfs_rq = task_cfs_rq(current); |
| 9119 | curr = cfs_rq->curr; |
Peter Zijlstra | e210bff | 2016-06-16 18:51:48 +0200 | [diff] [blame] | 9120 | if (curr) { |
| 9121 | update_curr(cfs_rq); |
Mike Galbraith | b5d9d73 | 2009-09-08 11:12:28 +0200 | [diff] [blame] | 9122 | se->vruntime = curr->vruntime; |
Peter Zijlstra | e210bff | 2016-06-16 18:51:48 +0200 | [diff] [blame] | 9123 | } |
Peter Zijlstra | aeb73b0 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 9124 | place_entity(cfs_rq, se, 1); |
Peter Zijlstra | 4d78e7b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 9125 | |
Peter Zijlstra | cd29fe6 | 2009-11-27 17:32:46 +0100 | [diff] [blame] | 9126 | if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) { |
Dmitry Adamushko | 87fefa3 | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 9127 | /* |
Ingo Molnar | edcb60a | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 9128 | * Upon rescheduling, sched_class::put_prev_task() will place |
| 9129 | * 'current' within the tree based on its new key value. |
| 9130 | */ |
Peter Zijlstra | 4d78e7b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 9131 | swap(curr->vruntime, se->vruntime); |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 9132 | resched_curr(rq); |
Peter Zijlstra | 4d78e7b | 2007-10-15 17:00:04 +0200 | [diff] [blame] | 9133 | } |
| 9134 | |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 9135 | se->vruntime -= cfs_rq->min_vruntime; |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9136 | rq_unlock(rq, &rf); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9137 | } |
| 9138 | |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9139 | /* |
| 9140 | * Priority of the task has changed. Check to see if we preempt |
| 9141 | * the current task. |
| 9142 | */ |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 9143 | static void |
| 9144 | prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9145 | { |
Kirill Tkhai | da0c1e6 | 2014-08-20 13:47:32 +0400 | [diff] [blame] | 9146 | if (!task_on_rq_queued(p)) |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 9147 | return; |
| 9148 | |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9149 | /* |
| 9150 | * Reschedule if we are currently running on this runqueue and |
| 9151 | * our priority decreased, or if we are not currently running on |
| 9152 | * this runqueue and our priority is higher than the current's |
| 9153 | */ |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 9154 | if (rq->curr == p) { |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9155 | if (p->prio > oldprio) |
Kirill Tkhai | 8875125 | 2014-06-29 00:03:57 +0400 | [diff] [blame] | 9156 | resched_curr(rq); |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9157 | } else |
Peter Zijlstra | 15afe09 | 2008-09-20 23:38:02 +0200 | [diff] [blame] | 9158 | check_preempt_curr(rq, p, 0); |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9159 | } |
| 9160 | |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9161 | static inline bool vruntime_normalized(struct task_struct *p) |
| 9162 | { |
| 9163 | struct sched_entity *se = &p->se; |
| 9164 | |
| 9165 | /* |
| 9166 | * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases, |
| 9167 | * the dequeue_entity(.flags=0) will already have normalized the |
| 9168 | * vruntime. |
| 9169 | */ |
| 9170 | if (p->on_rq) |
| 9171 | return true; |
| 9172 | |
| 9173 | /* |
| 9174 | * When !on_rq, vruntime of the task has usually NOT been normalized. |
| 9175 | * But there are some cases where it has already been normalized: |
| 9176 | * |
| 9177 | * - A forked child which is waiting for being woken up by |
| 9178 | * wake_up_new_task(). |
| 9179 | * - A task which has been woken up by try_to_wake_up() and |
| 9180 | * waiting for actually being woken up by sched_ttwu_pending(). |
| 9181 | */ |
| 9182 | if (!se->sum_exec_runtime || p->state == TASK_WAKING) |
| 9183 | return true; |
| 9184 | |
| 9185 | return false; |
| 9186 | } |
| 9187 | |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 9188 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 9189 | /* |
| 9190 | * Propagate the changes of the sched_entity across the tg tree to make it |
| 9191 | * visible to the root |
| 9192 | */ |
| 9193 | static void propagate_entity_cfs_rq(struct sched_entity *se) |
| 9194 | { |
| 9195 | struct cfs_rq *cfs_rq; |
| 9196 | |
| 9197 | /* Start to propagate at parent */ |
| 9198 | se = se->parent; |
| 9199 | |
| 9200 | for_each_sched_entity(se) { |
| 9201 | cfs_rq = cfs_rq_of(se); |
| 9202 | |
| 9203 | if (cfs_rq_throttled(cfs_rq)) |
| 9204 | break; |
| 9205 | |
| 9206 | update_load_avg(se, UPDATE_TG); |
| 9207 | } |
| 9208 | } |
| 9209 | #else |
| 9210 | static void propagate_entity_cfs_rq(struct sched_entity *se) { } |
| 9211 | #endif |
| 9212 | |
Vincent Guittot | df21791 | 2016-11-08 10:53:42 +0100 | [diff] [blame] | 9213 | static void detach_entity_cfs_rq(struct sched_entity *se) |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 9214 | { |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 9215 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 9216 | |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 9217 | /* Catch up with the cfs_rq and remove our load when we leave */ |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 9218 | update_load_avg(se, 0); |
Byungchul Park | a05e8c5 | 2015-08-20 20:21:56 +0900 | [diff] [blame] | 9219 | detach_entity_load_avg(cfs_rq, se); |
Peter Zijlstra | 7c3edd2 | 2016-07-13 10:56:25 +0200 | [diff] [blame] | 9220 | update_tg_load_avg(cfs_rq, false); |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 9221 | propagate_entity_cfs_rq(se); |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 9222 | } |
| 9223 | |
Vincent Guittot | df21791 | 2016-11-08 10:53:42 +0100 | [diff] [blame] | 9224 | static void attach_entity_cfs_rq(struct sched_entity *se) |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9225 | { |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9226 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
Byungchul Park | 7855a35 | 2015-08-10 18:02:55 +0900 | [diff] [blame] | 9227 | |
| 9228 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Michael wang | eb7a59b | 2014-02-20 11:14:53 +0800 | [diff] [blame] | 9229 | /* |
| 9230 | * Since the real-depth could have been changed (only FAIR |
| 9231 | * class maintain depth value), reset depth properly. |
| 9232 | */ |
| 9233 | se->depth = se->parent ? se->parent->depth + 1 : 0; |
| 9234 | #endif |
Byungchul Park | 7855a35 | 2015-08-10 18:02:55 +0900 | [diff] [blame] | 9235 | |
Vincent Guittot | df21791 | 2016-11-08 10:53:42 +0100 | [diff] [blame] | 9236 | /* Synchronize entity with its cfs_rq */ |
Vincent Guittot | d31b1a6 | 2016-11-08 10:53:44 +0100 | [diff] [blame] | 9237 | update_load_avg(se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9238 | attach_entity_load_avg(cfs_rq, se); |
Peter Zijlstra | 7c3edd2 | 2016-07-13 10:56:25 +0200 | [diff] [blame] | 9239 | update_tg_load_avg(cfs_rq, false); |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 9240 | propagate_entity_cfs_rq(se); |
Vincent Guittot | df21791 | 2016-11-08 10:53:42 +0100 | [diff] [blame] | 9241 | } |
| 9242 | |
| 9243 | static void detach_task_cfs_rq(struct task_struct *p) |
| 9244 | { |
| 9245 | struct sched_entity *se = &p->se; |
| 9246 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 9247 | |
| 9248 | if (!vruntime_normalized(p)) { |
| 9249 | /* |
| 9250 | * Fix up our vruntime so that the current sleep doesn't |
| 9251 | * cause 'unlimited' sleep bonus. |
| 9252 | */ |
| 9253 | place_entity(cfs_rq, se, 0); |
| 9254 | se->vruntime -= cfs_rq->min_vruntime; |
| 9255 | } |
| 9256 | |
| 9257 | detach_entity_cfs_rq(se); |
| 9258 | } |
| 9259 | |
| 9260 | static void attach_task_cfs_rq(struct task_struct *p) |
| 9261 | { |
| 9262 | struct sched_entity *se = &p->se; |
| 9263 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 9264 | |
| 9265 | attach_entity_cfs_rq(se); |
Byungchul Park | 6efdb10 | 2015-08-20 20:21:59 +0900 | [diff] [blame] | 9266 | |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9267 | if (!vruntime_normalized(p)) |
| 9268 | se->vruntime += cfs_rq->min_vruntime; |
| 9269 | } |
Byungchul Park | 7855a35 | 2015-08-10 18:02:55 +0900 | [diff] [blame] | 9270 | |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9271 | static void switched_from_fair(struct rq *rq, struct task_struct *p) |
| 9272 | { |
| 9273 | detach_task_cfs_rq(p); |
| 9274 | } |
| 9275 | |
| 9276 | static void switched_to_fair(struct rq *rq, struct task_struct *p) |
| 9277 | { |
| 9278 | attach_task_cfs_rq(p); |
| 9279 | |
| 9280 | if (task_on_rq_queued(p)) { |
Byungchul Park | 7855a35 | 2015-08-10 18:02:55 +0900 | [diff] [blame] | 9281 | /* |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9282 | * We were most likely switched from sched_rt, so |
| 9283 | * kick off the schedule if running, otherwise just see |
| 9284 | * if we can still preempt the current task. |
Byungchul Park | 7855a35 | 2015-08-10 18:02:55 +0900 | [diff] [blame] | 9285 | */ |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9286 | if (rq->curr == p) |
| 9287 | resched_curr(rq); |
| 9288 | else |
| 9289 | check_preempt_curr(rq, p, 0); |
Byungchul Park | 7855a35 | 2015-08-10 18:02:55 +0900 | [diff] [blame] | 9290 | } |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9291 | } |
| 9292 | |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 9293 | /* Account for a task changing its policy or group. |
| 9294 | * |
| 9295 | * This routine is mostly called to set cfs_rq->curr field when a task |
| 9296 | * migrates between groups/classes. |
| 9297 | */ |
| 9298 | static void set_curr_task_fair(struct rq *rq) |
| 9299 | { |
| 9300 | struct sched_entity *se = &rq->curr->se; |
| 9301 | |
Paul Turner | ec12cb7 | 2011-07-21 09:43:30 -0700 | [diff] [blame] | 9302 | for_each_sched_entity(se) { |
| 9303 | struct cfs_rq *cfs_rq = cfs_rq_of(se); |
| 9304 | |
| 9305 | set_next_entity(cfs_rq, se); |
| 9306 | /* ensure bandwidth has been allocated on our new cfs_rq */ |
| 9307 | account_cfs_rq_runtime(cfs_rq, 0); |
| 9308 | } |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 9309 | } |
| 9310 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9311 | void init_cfs_rq(struct cfs_rq *cfs_rq) |
| 9312 | { |
| 9313 | cfs_rq->tasks_timeline = RB_ROOT; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9314 | cfs_rq->min_vruntime = (u64)(-(1LL << 20)); |
| 9315 | #ifndef CONFIG_64BIT |
| 9316 | cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; |
| 9317 | #endif |
Alex Shi | 141965c | 2013-06-26 13:05:39 +0800 | [diff] [blame] | 9318 | #ifdef CONFIG_SMP |
Vincent Guittot | 09a43ac | 2016-11-08 10:53:45 +0100 | [diff] [blame] | 9319 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 9320 | cfs_rq->propagate_avg = 0; |
| 9321 | #endif |
Yuyang Du | 9d89c25 | 2015-07-15 08:04:37 +0800 | [diff] [blame] | 9322 | atomic_long_set(&cfs_rq->removed_load_avg, 0); |
| 9323 | atomic_long_set(&cfs_rq->removed_util_avg, 0); |
Paul Turner | 9ee474f | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 9324 | #endif |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9325 | } |
| 9326 | |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 9327 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Vincent Guittot | ea86cb4 | 2016-06-17 13:38:55 +0200 | [diff] [blame] | 9328 | static void task_set_group_fair(struct task_struct *p) |
| 9329 | { |
| 9330 | struct sched_entity *se = &p->se; |
| 9331 | |
| 9332 | set_task_rq(p, task_cpu(p)); |
| 9333 | se->depth = se->parent ? se->parent->depth + 1 : 0; |
| 9334 | } |
| 9335 | |
Peter Zijlstra | bc54da2 | 2015-08-31 17:13:55 +0200 | [diff] [blame] | 9336 | static void task_move_group_fair(struct task_struct *p) |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 9337 | { |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9338 | detach_task_cfs_rq(p); |
Peter Zijlstra | b2b5ce0 | 2010-10-15 15:24:15 +0200 | [diff] [blame] | 9339 | set_task_rq(p, task_cpu(p)); |
Byungchul Park | 6efdb10 | 2015-08-20 20:21:59 +0900 | [diff] [blame] | 9340 | |
| 9341 | #ifdef CONFIG_SMP |
| 9342 | /* Tell se's cfs_rq has been changed -- migrated */ |
| 9343 | p->se.avg.last_update_time = 0; |
| 9344 | #endif |
Byungchul Park | daa5940 | 2015-08-20 20:22:00 +0900 | [diff] [blame] | 9345 | attach_task_cfs_rq(p); |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 9346 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9347 | |
Vincent Guittot | ea86cb4 | 2016-06-17 13:38:55 +0200 | [diff] [blame] | 9348 | static void task_change_group_fair(struct task_struct *p, int type) |
| 9349 | { |
| 9350 | switch (type) { |
| 9351 | case TASK_SET_GROUP: |
| 9352 | task_set_group_fair(p); |
| 9353 | break; |
| 9354 | |
| 9355 | case TASK_MOVE_GROUP: |
| 9356 | task_move_group_fair(p); |
| 9357 | break; |
| 9358 | } |
| 9359 | } |
| 9360 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9361 | void free_fair_sched_group(struct task_group *tg) |
| 9362 | { |
| 9363 | int i; |
| 9364 | |
| 9365 | destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); |
| 9366 | |
| 9367 | for_each_possible_cpu(i) { |
| 9368 | if (tg->cfs_rq) |
| 9369 | kfree(tg->cfs_rq[i]); |
Peter Zijlstra | 6fe1f34 | 2016-01-21 22:24:16 +0100 | [diff] [blame] | 9370 | if (tg->se) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9371 | kfree(tg->se[i]); |
| 9372 | } |
| 9373 | |
| 9374 | kfree(tg->cfs_rq); |
| 9375 | kfree(tg->se); |
| 9376 | } |
| 9377 | |
| 9378 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) |
| 9379 | { |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9380 | struct sched_entity *se; |
Peter Zijlstra | b7fa30c | 2016-06-09 15:07:50 +0200 | [diff] [blame] | 9381 | struct cfs_rq *cfs_rq; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9382 | int i; |
| 9383 | |
| 9384 | tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL); |
| 9385 | if (!tg->cfs_rq) |
| 9386 | goto err; |
| 9387 | tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL); |
| 9388 | if (!tg->se) |
| 9389 | goto err; |
| 9390 | |
| 9391 | tg->shares = NICE_0_LOAD; |
| 9392 | |
| 9393 | init_cfs_bandwidth(tg_cfs_bandwidth(tg)); |
| 9394 | |
| 9395 | for_each_possible_cpu(i) { |
| 9396 | cfs_rq = kzalloc_node(sizeof(struct cfs_rq), |
| 9397 | GFP_KERNEL, cpu_to_node(i)); |
| 9398 | if (!cfs_rq) |
| 9399 | goto err; |
| 9400 | |
| 9401 | se = kzalloc_node(sizeof(struct sched_entity), |
| 9402 | GFP_KERNEL, cpu_to_node(i)); |
| 9403 | if (!se) |
| 9404 | goto err_free_rq; |
| 9405 | |
| 9406 | init_cfs_rq(cfs_rq); |
| 9407 | init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); |
Yuyang Du | 540247f | 2015-07-15 08:04:39 +0800 | [diff] [blame] | 9408 | init_entity_runnable_average(se); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9409 | } |
| 9410 | |
| 9411 | return 1; |
| 9412 | |
| 9413 | err_free_rq: |
| 9414 | kfree(cfs_rq); |
| 9415 | err: |
| 9416 | return 0; |
| 9417 | } |
| 9418 | |
Peter Zijlstra | 8663e24 | 2016-06-22 14:58:02 +0200 | [diff] [blame] | 9419 | void online_fair_sched_group(struct task_group *tg) |
| 9420 | { |
| 9421 | struct sched_entity *se; |
| 9422 | struct rq *rq; |
| 9423 | int i; |
| 9424 | |
| 9425 | for_each_possible_cpu(i) { |
| 9426 | rq = cpu_rq(i); |
| 9427 | se = tg->se[i]; |
| 9428 | |
| 9429 | raw_spin_lock_irq(&rq->lock); |
Peter Zijlstra | 4126bad | 2016-10-03 16:20:59 +0200 | [diff] [blame] | 9430 | update_rq_clock(rq); |
Vincent Guittot | d032669 | 2016-11-08 10:53:47 +0100 | [diff] [blame] | 9431 | attach_entity_cfs_rq(se); |
Peter Zijlstra | 55e16d3 | 2016-06-22 15:14:26 +0200 | [diff] [blame] | 9432 | sync_throttle(tg, i); |
Peter Zijlstra | 8663e24 | 2016-06-22 14:58:02 +0200 | [diff] [blame] | 9433 | raw_spin_unlock_irq(&rq->lock); |
| 9434 | } |
| 9435 | } |
| 9436 | |
Peter Zijlstra | 6fe1f34 | 2016-01-21 22:24:16 +0100 | [diff] [blame] | 9437 | void unregister_fair_sched_group(struct task_group *tg) |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9438 | { |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9439 | unsigned long flags; |
Peter Zijlstra | 6fe1f34 | 2016-01-21 22:24:16 +0100 | [diff] [blame] | 9440 | struct rq *rq; |
| 9441 | int cpu; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9442 | |
Peter Zijlstra | 6fe1f34 | 2016-01-21 22:24:16 +0100 | [diff] [blame] | 9443 | for_each_possible_cpu(cpu) { |
| 9444 | if (tg->se[cpu]) |
| 9445 | remove_entity_load_avg(tg->se[cpu]); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9446 | |
Peter Zijlstra | 6fe1f34 | 2016-01-21 22:24:16 +0100 | [diff] [blame] | 9447 | /* |
| 9448 | * Only empty task groups can be destroyed; so we can speculatively |
| 9449 | * check on_list without danger of it being re-added. |
| 9450 | */ |
| 9451 | if (!tg->cfs_rq[cpu]->on_list) |
| 9452 | continue; |
| 9453 | |
| 9454 | rq = cpu_rq(cpu); |
| 9455 | |
| 9456 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 9457 | list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); |
| 9458 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 9459 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9460 | } |
| 9461 | |
| 9462 | void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, |
| 9463 | struct sched_entity *se, int cpu, |
| 9464 | struct sched_entity *parent) |
| 9465 | { |
| 9466 | struct rq *rq = cpu_rq(cpu); |
| 9467 | |
| 9468 | cfs_rq->tg = tg; |
| 9469 | cfs_rq->rq = rq; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9470 | init_cfs_rq_runtime(cfs_rq); |
| 9471 | |
| 9472 | tg->cfs_rq[cpu] = cfs_rq; |
| 9473 | tg->se[cpu] = se; |
| 9474 | |
| 9475 | /* se could be NULL for root_task_group */ |
| 9476 | if (!se) |
| 9477 | return; |
| 9478 | |
Peter Zijlstra | fed14d4 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 9479 | if (!parent) { |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9480 | se->cfs_rq = &rq->cfs; |
Peter Zijlstra | fed14d4 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 9481 | se->depth = 0; |
| 9482 | } else { |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9483 | se->cfs_rq = parent->my_q; |
Peter Zijlstra | fed14d4 | 2012-02-11 06:05:00 +0100 | [diff] [blame] | 9484 | se->depth = parent->depth + 1; |
| 9485 | } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9486 | |
| 9487 | se->my_q = cfs_rq; |
Paul Turner | 0ac9b1c | 2013-10-16 11:16:27 -0700 | [diff] [blame] | 9488 | /* guarantee group entities always have weight */ |
| 9489 | update_load_set(&se->load, NICE_0_LOAD); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9490 | se->parent = parent; |
| 9491 | } |
| 9492 | |
| 9493 | static DEFINE_MUTEX(shares_mutex); |
| 9494 | |
| 9495 | int sched_group_set_shares(struct task_group *tg, unsigned long shares) |
| 9496 | { |
| 9497 | int i; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9498 | |
| 9499 | /* |
| 9500 | * We can't change the weight of the root cgroup. |
| 9501 | */ |
| 9502 | if (!tg->se[0]) |
| 9503 | return -EINVAL; |
| 9504 | |
| 9505 | shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); |
| 9506 | |
| 9507 | mutex_lock(&shares_mutex); |
| 9508 | if (tg->shares == shares) |
| 9509 | goto done; |
| 9510 | |
| 9511 | tg->shares = shares; |
| 9512 | for_each_possible_cpu(i) { |
| 9513 | struct rq *rq = cpu_rq(i); |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9514 | struct sched_entity *se = tg->se[i]; |
| 9515 | struct rq_flags rf; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9516 | |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9517 | /* Propagate contribution to hierarchy */ |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9518 | rq_lock_irqsave(rq, &rf); |
Frederic Weisbecker | 71b1da4 | 2013-04-12 01:50:59 +0200 | [diff] [blame] | 9519 | update_rq_clock(rq); |
Vincent Guittot | 89ee048 | 2016-12-21 16:50:26 +0100 | [diff] [blame] | 9520 | for_each_sched_entity(se) { |
| 9521 | update_load_avg(se, UPDATE_TG); |
| 9522 | update_cfs_shares(se); |
| 9523 | } |
Peter Zijlstra | 8a8c69c | 2016-10-04 16:04:35 +0200 | [diff] [blame] | 9524 | rq_unlock_irqrestore(rq, &rf); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9525 | } |
| 9526 | |
| 9527 | done: |
| 9528 | mutex_unlock(&shares_mutex); |
| 9529 | return 0; |
| 9530 | } |
| 9531 | #else /* CONFIG_FAIR_GROUP_SCHED */ |
| 9532 | |
| 9533 | void free_fair_sched_group(struct task_group *tg) { } |
| 9534 | |
| 9535 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) |
| 9536 | { |
| 9537 | return 1; |
| 9538 | } |
| 9539 | |
Peter Zijlstra | 8663e24 | 2016-06-22 14:58:02 +0200 | [diff] [blame] | 9540 | void online_fair_sched_group(struct task_group *tg) { } |
| 9541 | |
Peter Zijlstra | 6fe1f34 | 2016-01-21 22:24:16 +0100 | [diff] [blame] | 9542 | void unregister_fair_sched_group(struct task_group *tg) { } |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9543 | |
| 9544 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
| 9545 | |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 9546 | |
H Hartley Sweeten | 6d686f4 | 2010-01-13 20:21:52 -0700 | [diff] [blame] | 9547 | static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) |
Peter Williams | 0d721ce | 2009-09-21 01:31:53 +0000 | [diff] [blame] | 9548 | { |
| 9549 | struct sched_entity *se = &task->se; |
Peter Williams | 0d721ce | 2009-09-21 01:31:53 +0000 | [diff] [blame] | 9550 | unsigned int rr_interval = 0; |
| 9551 | |
| 9552 | /* |
| 9553 | * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise |
| 9554 | * idle runqueue: |
| 9555 | */ |
Peter Williams | 0d721ce | 2009-09-21 01:31:53 +0000 | [diff] [blame] | 9556 | if (rq->cfs.load.weight) |
Zhu Yanhai | a59f4e0 | 2013-01-08 12:56:52 +0800 | [diff] [blame] | 9557 | rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se)); |
Peter Williams | 0d721ce | 2009-09-21 01:31:53 +0000 | [diff] [blame] | 9558 | |
| 9559 | return rr_interval; |
| 9560 | } |
| 9561 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9562 | /* |
| 9563 | * All the scheduling class methods: |
| 9564 | */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9565 | const struct sched_class fair_sched_class = { |
Ingo Molnar | 5522d5d | 2007-10-15 17:00:12 +0200 | [diff] [blame] | 9566 | .next = &idle_sched_class, |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9567 | .enqueue_task = enqueue_task_fair, |
| 9568 | .dequeue_task = dequeue_task_fair, |
| 9569 | .yield_task = yield_task_fair, |
Mike Galbraith | d95f412 | 2011-02-01 09:50:51 -0500 | [diff] [blame] | 9570 | .yield_to_task = yield_to_task_fair, |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9571 | |
Ingo Molnar | 2e09bf5 | 2007-10-15 17:00:05 +0200 | [diff] [blame] | 9572 | .check_preempt_curr = check_preempt_wakeup, |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9573 | |
| 9574 | .pick_next_task = pick_next_task_fair, |
| 9575 | .put_prev_task = put_prev_task_fair, |
| 9576 | |
Peter Williams | 681f3e6 | 2007-10-24 18:23:51 +0200 | [diff] [blame] | 9577 | #ifdef CONFIG_SMP |
Li Zefan | 4ce72a2 | 2008-10-22 15:25:26 +0800 | [diff] [blame] | 9578 | .select_task_rq = select_task_rq_fair, |
Paul Turner | 0a74bef | 2012-10-04 13:18:30 +0200 | [diff] [blame] | 9579 | .migrate_task_rq = migrate_task_rq_fair, |
Alex Shi | 141965c | 2013-06-26 13:05:39 +0800 | [diff] [blame] | 9580 | |
Christian Ehrhardt | 0bcdcf2 | 2009-11-30 12:16:46 +0100 | [diff] [blame] | 9581 | .rq_online = rq_online_fair, |
| 9582 | .rq_offline = rq_offline_fair, |
Peter Zijlstra | 88ec22d | 2009-12-16 18:04:41 +0100 | [diff] [blame] | 9583 | |
Yuyang Du | 1269557 | 2015-07-15 08:04:40 +0800 | [diff] [blame] | 9584 | .task_dead = task_dead_fair, |
Peter Zijlstra | c5b2803 | 2015-05-15 17:43:35 +0200 | [diff] [blame] | 9585 | .set_cpus_allowed = set_cpus_allowed_common, |
Peter Williams | 681f3e6 | 2007-10-24 18:23:51 +0200 | [diff] [blame] | 9586 | #endif |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9587 | |
Srivatsa Vaddagiri | 83b699e | 2007-10-15 17:00:08 +0200 | [diff] [blame] | 9588 | .set_curr_task = set_curr_task_fair, |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9589 | .task_tick = task_tick_fair, |
Peter Zijlstra | cd29fe6 | 2009-11-27 17:32:46 +0100 | [diff] [blame] | 9590 | .task_fork = task_fork_fair, |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9591 | |
| 9592 | .prio_changed = prio_changed_fair, |
Peter Zijlstra | da7a735 | 2011-01-17 17:03:27 +0100 | [diff] [blame] | 9593 | .switched_from = switched_from_fair, |
Steven Rostedt | cb46984 | 2008-01-25 21:08:22 +0100 | [diff] [blame] | 9594 | .switched_to = switched_to_fair, |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 9595 | |
Peter Williams | 0d721ce | 2009-09-21 01:31:53 +0000 | [diff] [blame] | 9596 | .get_rr_interval = get_rr_interval_fair, |
| 9597 | |
Stanislaw Gruszka | 6e99891 | 2014-11-12 16:58:44 +0100 | [diff] [blame] | 9598 | .update_curr = update_curr_fair, |
| 9599 | |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 9600 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Vincent Guittot | ea86cb4 | 2016-06-17 13:38:55 +0200 | [diff] [blame] | 9601 | .task_change_group = task_change_group_fair, |
Peter Zijlstra | 810b381 | 2008-02-29 15:21:01 -0500 | [diff] [blame] | 9602 | #endif |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9603 | }; |
| 9604 | |
| 9605 | #ifdef CONFIG_SCHED_DEBUG |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9606 | void print_cfs_stats(struct seq_file *m, int cpu) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9607 | { |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 9608 | struct cfs_rq *cfs_rq, *pos; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9609 | |
Peter Zijlstra | 5973e5b | 2008-01-25 21:08:34 +0100 | [diff] [blame] | 9610 | rcu_read_lock(); |
Tejun Heo | a9e7f65 | 2017-04-25 17:43:50 -0700 | [diff] [blame] | 9611 | for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) |
Ingo Molnar | 5cef9ec | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 9612 | print_cfs_rq(m, cpu, cfs_rq); |
Peter Zijlstra | 5973e5b | 2008-01-25 21:08:34 +0100 | [diff] [blame] | 9613 | rcu_read_unlock(); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 9614 | } |
Srikar Dronamraju | 397f237 | 2015-06-25 22:51:43 +0530 | [diff] [blame] | 9615 | |
| 9616 | #ifdef CONFIG_NUMA_BALANCING |
| 9617 | void show_numa_stats(struct task_struct *p, struct seq_file *m) |
| 9618 | { |
| 9619 | int node; |
| 9620 | unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; |
| 9621 | |
| 9622 | for_each_online_node(node) { |
| 9623 | if (p->numa_faults) { |
| 9624 | tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; |
| 9625 | tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; |
| 9626 | } |
| 9627 | if (p->numa_group) { |
| 9628 | gsf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 0)], |
| 9629 | gpf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 1)]; |
| 9630 | } |
| 9631 | print_numa_stats(m, node, tsf, tpf, gsf, gpf); |
| 9632 | } |
| 9633 | } |
| 9634 | #endif /* CONFIG_NUMA_BALANCING */ |
| 9635 | #endif /* CONFIG_SCHED_DEBUG */ |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9636 | |
| 9637 | __init void init_sched_fair_class(void) |
| 9638 | { |
| 9639 | #ifdef CONFIG_SMP |
| 9640 | open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); |
| 9641 | |
Frederic Weisbecker | 3451d02 | 2011-08-10 23:21:01 +0200 | [diff] [blame] | 9642 | #ifdef CONFIG_NO_HZ_COMMON |
Diwakar Tundlam | 554ceca | 2012-03-07 14:44:26 -0800 | [diff] [blame] | 9643 | nohz.next_balance = jiffies; |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9644 | zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); |
Peter Zijlstra | 029632f | 2011-10-25 10:00:11 +0200 | [diff] [blame] | 9645 | #endif |
| 9646 | #endif /* SMP */ |
| 9647 | |
| 9648 | } |