blob: b024106daa70163646b31a031085b4d732b4f096 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/timer.c
3 *
john stultz85240702007-05-08 00:27:59 -07004 * Kernel internal timers, basic process system calls
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * Copyright (C) 1991, 1992 Linus Torvalds
7 *
8 * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
9 *
10 * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
11 * "A Kernel Model for Precision Timekeeping" by Dave Mills
12 * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
13 * serialize accesses to xtime/lost_ticks).
14 * Copyright (C) 1998 Andrea Arcangeli
15 * 1999-03-10 Improved NTP compatibility by Ulrich Windl
16 * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
17 * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
18 * Copyright (C) 2000, 2001, 2002 Ingo Molnar
19 * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
20 */
21
22#include <linux/kernel_stat.h>
23#include <linux/module.h>
24#include <linux/interrupt.h>
25#include <linux/percpu.h>
26#include <linux/init.h>
27#include <linux/mm.h>
28#include <linux/swap.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070029#include <linux/pid_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/notifier.h>
31#include <linux/thread_info.h>
32#include <linux/time.h>
33#include <linux/jiffies.h>
34#include <linux/posix-timers.h>
35#include <linux/cpu.h>
36#include <linux/syscalls.h>
Adrian Bunk97a41e22006-01-08 01:02:17 -080037#include <linux/delay.h>
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080038#include <linux/tick.h>
Ingo Molnar82f67cd2007-02-16 01:28:13 -080039#include <linux/kallsyms.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
41#include <asm/uaccess.h>
42#include <asm/unistd.h>
43#include <asm/div64.h>
44#include <asm/timex.h>
45#include <asm/io.h>
46
Thomas Gleixnerecea8d12005-10-30 15:03:00 -080047u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
48
49EXPORT_SYMBOL(jiffies_64);
50
Linus Torvalds1da177e2005-04-16 15:20:36 -070051/*
52 * per-CPU timer vector definitions:
53 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070054#define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)
55#define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)
56#define TVN_SIZE (1 << TVN_BITS)
57#define TVR_SIZE (1 << TVR_BITS)
58#define TVN_MASK (TVN_SIZE - 1)
59#define TVR_MASK (TVR_SIZE - 1)
60
Pavel Macheka6fa8e52008-01-30 13:30:00 +010061struct tvec {
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 struct list_head vec[TVN_SIZE];
Pavel Macheka6fa8e52008-01-30 13:30:00 +010063};
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Pavel Macheka6fa8e52008-01-30 13:30:00 +010065struct tvec_root {
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 struct list_head vec[TVR_SIZE];
Pavel Macheka6fa8e52008-01-30 13:30:00 +010067};
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
Pavel Macheka6fa8e52008-01-30 13:30:00 +010069struct tvec_base {
Oleg Nesterov3691c512006-03-31 02:30:30 -080070 spinlock_t lock;
71 struct timer_list *running_timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 unsigned long timer_jiffies;
Pavel Macheka6fa8e52008-01-30 13:30:00 +010073 struct tvec_root tv1;
74 struct tvec tv2;
75 struct tvec tv3;
76 struct tvec tv4;
77 struct tvec tv5;
Venki Pallipadi6e453a62007-05-08 00:27:44 -070078} ____cacheline_aligned;
Linus Torvalds1da177e2005-04-16 15:20:36 -070079
Pavel Macheka6fa8e52008-01-30 13:30:00 +010080struct tvec_base boot_tvec_bases;
Oleg Nesterov3691c512006-03-31 02:30:30 -080081EXPORT_SYMBOL(boot_tvec_bases);
Pavel Macheka6fa8e52008-01-30 13:30:00 +010082static DEFINE_PER_CPU(struct tvec_base *, tvec_bases) = &boot_tvec_bases;
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
Venki Pallipadi6e453a62007-05-08 00:27:44 -070084/*
Pavel Macheka6fa8e52008-01-30 13:30:00 +010085 * Note that all tvec_bases are 2 byte aligned and lower bit of
Venki Pallipadi6e453a62007-05-08 00:27:44 -070086 * base in timer_list is guaranteed to be zero. Use the LSB for
87 * the new flag to indicate whether the timer is deferrable
88 */
89#define TBASE_DEFERRABLE_FLAG (0x1)
90
91/* Functions below help us manage 'deferrable' flag */
Pavel Macheka6fa8e52008-01-30 13:30:00 +010092static inline unsigned int tbase_get_deferrable(struct tvec_base *base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -070093{
akpm@linux-foundation.orge9910842007-05-10 03:16:01 -070094 return ((unsigned int)(unsigned long)base & TBASE_DEFERRABLE_FLAG);
Venki Pallipadi6e453a62007-05-08 00:27:44 -070095}
96
Pavel Macheka6fa8e52008-01-30 13:30:00 +010097static inline struct tvec_base *tbase_get_base(struct tvec_base *base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -070098{
Pavel Macheka6fa8e52008-01-30 13:30:00 +010099 return ((struct tvec_base *)((unsigned long)base & ~TBASE_DEFERRABLE_FLAG));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700100}
101
102static inline void timer_set_deferrable(struct timer_list *timer)
103{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100104 timer->base = ((struct tvec_base *)((unsigned long)(timer->base) |
Thomas Gleixner68194572007-07-19 01:49:16 -0700105 TBASE_DEFERRABLE_FLAG));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700106}
107
108static inline void
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100109timer_set_base(struct timer_list *timer, struct tvec_base *new_base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700110{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100111 timer->base = (struct tvec_base *)((unsigned long)(new_base) |
Thomas Gleixner68194572007-07-19 01:49:16 -0700112 tbase_get_deferrable(timer->base));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700113}
114
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800115/**
116 * __round_jiffies - function to round jiffies to a full second
117 * @j: the time in (absolute) jiffies that should be rounded
118 * @cpu: the processor number on which the timeout will happen
119 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800120 * __round_jiffies() rounds an absolute time in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800121 * up or down to (approximately) full seconds. This is useful for timers
122 * for which the exact time they fire does not matter too much, as long as
123 * they fire approximately every X seconds.
124 *
125 * By rounding these timers to whole seconds, all such timers will fire
126 * at the same time, rather than at various times spread out. The goal
127 * of this is to have the CPU wake up less, which saves power.
128 *
129 * The exact rounding is skewed for each processor to avoid all
130 * processors firing at the exact same time, which could lead
131 * to lock contention or spurious cache line bouncing.
132 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800133 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800134 */
135unsigned long __round_jiffies(unsigned long j, int cpu)
136{
137 int rem;
138 unsigned long original = j;
139
140 /*
141 * We don't want all cpus firing their timers at once hitting the
142 * same lock or cachelines, so we skew each extra cpu with an extra
143 * 3 jiffies. This 3 jiffies came originally from the mm/ code which
144 * already did this.
145 * The skew is done by adding 3*cpunr, then round, then subtract this
146 * extra offset again.
147 */
148 j += cpu * 3;
149
150 rem = j % HZ;
151
152 /*
153 * If the target jiffie is just after a whole second (which can happen
154 * due to delays of the timer irq, long irq off times etc etc) then
155 * we should round down to the whole second, not up. Use 1/4th second
156 * as cutoff for this rounding as an extreme upper bound for this.
157 */
158 if (rem < HZ/4) /* round down */
159 j = j - rem;
160 else /* round up */
161 j = j - rem + HZ;
162
163 /* now that we have rounded, subtract the extra skew again */
164 j -= cpu * 3;
165
166 if (j <= jiffies) /* rounding ate our timeout entirely; */
167 return original;
168 return j;
169}
170EXPORT_SYMBOL_GPL(__round_jiffies);
171
172/**
173 * __round_jiffies_relative - function to round jiffies to a full second
174 * @j: the time in (relative) jiffies that should be rounded
175 * @cpu: the processor number on which the timeout will happen
176 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800177 * __round_jiffies_relative() rounds a time delta in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800178 * up or down to (approximately) full seconds. This is useful for timers
179 * for which the exact time they fire does not matter too much, as long as
180 * they fire approximately every X seconds.
181 *
182 * By rounding these timers to whole seconds, all such timers will fire
183 * at the same time, rather than at various times spread out. The goal
184 * of this is to have the CPU wake up less, which saves power.
185 *
186 * The exact rounding is skewed for each processor to avoid all
187 * processors firing at the exact same time, which could lead
188 * to lock contention or spurious cache line bouncing.
189 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800190 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800191 */
192unsigned long __round_jiffies_relative(unsigned long j, int cpu)
193{
194 /*
195 * In theory the following code can skip a jiffy in case jiffies
196 * increments right between the addition and the later subtraction.
197 * However since the entire point of this function is to use approximate
198 * timeouts, it's entirely ok to not handle that.
199 */
200 return __round_jiffies(j + jiffies, cpu) - jiffies;
201}
202EXPORT_SYMBOL_GPL(__round_jiffies_relative);
203
204/**
205 * round_jiffies - function to round jiffies to a full second
206 * @j: the time in (absolute) jiffies that should be rounded
207 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800208 * round_jiffies() rounds an absolute time in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800209 * up or down to (approximately) full seconds. This is useful for timers
210 * for which the exact time they fire does not matter too much, as long as
211 * they fire approximately every X seconds.
212 *
213 * By rounding these timers to whole seconds, all such timers will fire
214 * at the same time, rather than at various times spread out. The goal
215 * of this is to have the CPU wake up less, which saves power.
216 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800217 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800218 */
219unsigned long round_jiffies(unsigned long j)
220{
221 return __round_jiffies(j, raw_smp_processor_id());
222}
223EXPORT_SYMBOL_GPL(round_jiffies);
224
225/**
226 * round_jiffies_relative - function to round jiffies to a full second
227 * @j: the time in (relative) jiffies that should be rounded
228 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800229 * round_jiffies_relative() rounds a time delta in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800230 * up or down to (approximately) full seconds. This is useful for timers
231 * for which the exact time they fire does not matter too much, as long as
232 * they fire approximately every X seconds.
233 *
234 * By rounding these timers to whole seconds, all such timers will fire
235 * at the same time, rather than at various times spread out. The goal
236 * of this is to have the CPU wake up less, which saves power.
237 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800238 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800239 */
240unsigned long round_jiffies_relative(unsigned long j)
241{
242 return __round_jiffies_relative(j, raw_smp_processor_id());
243}
244EXPORT_SYMBOL_GPL(round_jiffies_relative);
245
246
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100247static inline void set_running_timer(struct tvec_base *base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 struct timer_list *timer)
249{
250#ifdef CONFIG_SMP
Oleg Nesterov3691c512006-03-31 02:30:30 -0800251 base->running_timer = timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252#endif
253}
254
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100255static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256{
257 unsigned long expires = timer->expires;
258 unsigned long idx = expires - base->timer_jiffies;
259 struct list_head *vec;
260
261 if (idx < TVR_SIZE) {
262 int i = expires & TVR_MASK;
263 vec = base->tv1.vec + i;
264 } else if (idx < 1 << (TVR_BITS + TVN_BITS)) {
265 int i = (expires >> TVR_BITS) & TVN_MASK;
266 vec = base->tv2.vec + i;
267 } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {
268 int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;
269 vec = base->tv3.vec + i;
270 } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {
271 int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;
272 vec = base->tv4.vec + i;
273 } else if ((signed long) idx < 0) {
274 /*
275 * Can happen if you add a timer with expires == jiffies,
276 * or you set a timer to go off in the past
277 */
278 vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);
279 } else {
280 int i;
281 /* If the timeout is larger than 0xffffffff on 64-bit
282 * architectures then we use the maximum timeout:
283 */
284 if (idx > 0xffffffffUL) {
285 idx = 0xffffffffUL;
286 expires = idx + base->timer_jiffies;
287 }
288 i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;
289 vec = base->tv5.vec + i;
290 }
291 /*
292 * Timers are FIFO:
293 */
294 list_add_tail(&timer->entry, vec);
295}
296
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800297#ifdef CONFIG_TIMER_STATS
298void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr)
299{
300 if (timer->start_site)
301 return;
302
303 timer->start_site = addr;
304 memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
305 timer->start_pid = current->pid;
306}
Venki Pallipadic5c061b82007-07-15 23:40:30 -0700307
308static void timer_stats_account_timer(struct timer_list *timer)
309{
310 unsigned int flag = 0;
311
312 if (unlikely(tbase_get_deferrable(timer->base)))
313 flag |= TIMER_STATS_FLAG_DEFERRABLE;
314
315 timer_stats_update_stats(timer, timer->start_pid, timer->start_site,
316 timer->function, timer->start_comm, flag);
317}
318
319#else
320static void timer_stats_account_timer(struct timer_list *timer) {}
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800321#endif
322
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700323/**
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700324 * init_timer - initialize a timer.
325 * @timer: the timer to be initialized
326 *
327 * init_timer() must be done to a timer prior calling *any* of the
328 * other timer functions.
329 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800330void init_timer(struct timer_list *timer)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700331{
332 timer->entry.next = NULL;
Paul Mackerrasbfe5d832006-06-25 05:47:14 -0700333 timer->base = __raw_get_cpu_var(tvec_bases);
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800334#ifdef CONFIG_TIMER_STATS
335 timer->start_site = NULL;
336 timer->start_pid = -1;
337 memset(timer->start_comm, 0, TASK_COMM_LEN);
338#endif
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700339}
340EXPORT_SYMBOL(init_timer);
341
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800342void init_timer_deferrable(struct timer_list *timer)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700343{
344 init_timer(timer);
345 timer_set_deferrable(timer);
346}
347EXPORT_SYMBOL(init_timer_deferrable);
348
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700349static inline void detach_timer(struct timer_list *timer,
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800350 int clear_pending)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700351{
352 struct list_head *entry = &timer->entry;
353
354 __list_del(entry->prev, entry->next);
355 if (clear_pending)
356 entry->next = NULL;
357 entry->prev = LIST_POISON2;
358}
359
360/*
Oleg Nesterov3691c512006-03-31 02:30:30 -0800361 * We are using hashed locking: holding per_cpu(tvec_bases).lock
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700362 * means that all timers which are tied to this base via timer->base are
363 * locked, and the base itself is locked too.
364 *
365 * So __run_timers/migrate_timers can safely modify all timers which could
366 * be found on ->tvX lists.
367 *
368 * When the timer's base is locked, and the timer removed from list, it is
369 * possible to set timer->base = NULL and drop the lock: the timer remains
370 * locked.
371 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100372static struct tvec_base *lock_timer_base(struct timer_list *timer,
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700373 unsigned long *flags)
Josh Triplett89e7e3742006-09-29 01:59:36 -0700374 __acquires(timer->base->lock)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700375{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100376 struct tvec_base *base;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700377
378 for (;;) {
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100379 struct tvec_base *prelock_base = timer->base;
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700380 base = tbase_get_base(prelock_base);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700381 if (likely(base != NULL)) {
382 spin_lock_irqsave(&base->lock, *flags);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700383 if (likely(prelock_base == timer->base))
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700384 return base;
385 /* The timer has migrated to another CPU */
386 spin_unlock_irqrestore(&base->lock, *flags);
387 }
388 cpu_relax();
389 }
390}
391
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392int __mod_timer(struct timer_list *timer, unsigned long expires)
393{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100394 struct tvec_base *base, *new_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 unsigned long flags;
396 int ret = 0;
397
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800398 timer_stats_timer_set_start_info(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 BUG_ON(!timer->function);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700401 base = lock_timer_base(timer, &flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700403 if (timer_pending(timer)) {
404 detach_timer(timer, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 ret = 1;
406 }
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700407
Jan Beulicha4a61982006-03-24 03:15:54 -0800408 new_base = __get_cpu_var(tvec_bases);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700409
Oleg Nesterov3691c512006-03-31 02:30:30 -0800410 if (base != new_base) {
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700411 /*
412 * We are trying to schedule the timer on the local CPU.
413 * However we can't change timer's base while it is running,
414 * otherwise del_timer_sync() can't detect that the timer's
415 * handler yet has not finished. This also guarantees that
416 * the timer is serialized wrt itself.
417 */
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800418 if (likely(base->running_timer != timer)) {
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700419 /* See the comment in lock_timer_base() */
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700420 timer_set_base(timer, NULL);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700421 spin_unlock(&base->lock);
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800422 base = new_base;
423 spin_lock(&base->lock);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700424 timer_set_base(timer, base);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700425 }
426 }
427
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 timer->expires = expires;
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800429 internal_add_timer(base, timer);
430 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431
432 return ret;
433}
434
435EXPORT_SYMBOL(__mod_timer);
436
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700437/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438 * add_timer_on - start a timer on a particular CPU
439 * @timer: the timer to be added
440 * @cpu: the CPU to start it on
441 *
442 * This is not very scalable on SMP. Double adds are not possible.
443 */
444void add_timer_on(struct timer_list *timer, int cpu)
445{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100446 struct tvec_base *base = per_cpu(tvec_bases, cpu);
Thomas Gleixner68194572007-07-19 01:49:16 -0700447 unsigned long flags;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700448
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800449 timer_stats_timer_set_start_info(timer);
Thomas Gleixner68194572007-07-19 01:49:16 -0700450 BUG_ON(timer_pending(timer) || !timer->function);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800451 spin_lock_irqsave(&base->lock, flags);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700452 timer_set_base(timer, base);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 internal_add_timer(base, timer);
Thomas Gleixner06d83082008-03-22 09:20:24 +0100454 /*
455 * Check whether the other CPU is idle and needs to be
456 * triggered to reevaluate the timer wheel when nohz is
457 * active. We are protected against the other CPU fiddling
458 * with the timer by holding the timer base lock. This also
459 * makes sure that a CPU on the way to idle can not evaluate
460 * the timer wheel.
461 */
462 wake_up_idle_cpu(cpu);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800463 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464}
465
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700466/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 * mod_timer - modify a timer's timeout
468 * @timer: the timer to be modified
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700469 * @expires: new timeout in jiffies
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800471 * mod_timer() is a more efficient way to update the expire field of an
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 * active timer (if the timer is inactive it will be activated)
473 *
474 * mod_timer(timer, expires) is equivalent to:
475 *
476 * del_timer(timer); timer->expires = expires; add_timer(timer);
477 *
478 * Note that if there are multiple unserialized concurrent users of the
479 * same timer, then mod_timer() is the only safe way to modify the timeout,
480 * since add_timer() cannot modify an already running timer.
481 *
482 * The function returns whether it has modified a pending timer or not.
483 * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an
484 * active timer returns 1.)
485 */
486int mod_timer(struct timer_list *timer, unsigned long expires)
487{
488 BUG_ON(!timer->function);
489
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800490 timer_stats_timer_set_start_info(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 /*
492 * This is a common optimization triggered by the
493 * networking code - if the timer is re-modified
494 * to be the same thing then just return:
495 */
496 if (timer->expires == expires && timer_pending(timer))
497 return 1;
498
499 return __mod_timer(timer, expires);
500}
501
502EXPORT_SYMBOL(mod_timer);
503
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700504/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 * del_timer - deactive a timer.
506 * @timer: the timer to be deactivated
507 *
508 * del_timer() deactivates a timer - this works on both active and inactive
509 * timers.
510 *
511 * The function returns whether it has deactivated a pending timer or not.
512 * (ie. del_timer() of an inactive timer returns 0, del_timer() of an
513 * active timer returns 1.)
514 */
515int del_timer(struct timer_list *timer)
516{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100517 struct tvec_base *base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 unsigned long flags;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700519 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800521 timer_stats_timer_clear_start_info(timer);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700522 if (timer_pending(timer)) {
523 base = lock_timer_base(timer, &flags);
524 if (timer_pending(timer)) {
525 detach_timer(timer, 1);
526 ret = 1;
527 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700531 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532}
533
534EXPORT_SYMBOL(del_timer);
535
536#ifdef CONFIG_SMP
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700537/**
538 * try_to_del_timer_sync - Try to deactivate a timer
539 * @timer: timer do del
540 *
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700541 * This function tries to deactivate a timer. Upon successful (ret >= 0)
542 * exit the timer is not queued and the handler is not running on any CPU.
543 *
544 * It must not be called from interrupt contexts.
545 */
546int try_to_del_timer_sync(struct timer_list *timer)
547{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100548 struct tvec_base *base;
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700549 unsigned long flags;
550 int ret = -1;
551
552 base = lock_timer_base(timer, &flags);
553
554 if (base->running_timer == timer)
555 goto out;
556
557 ret = 0;
558 if (timer_pending(timer)) {
559 detach_timer(timer, 1);
560 ret = 1;
561 }
562out:
563 spin_unlock_irqrestore(&base->lock, flags);
564
565 return ret;
566}
567
David Howellse19dff12007-04-26 15:46:56 -0700568EXPORT_SYMBOL(try_to_del_timer_sync);
569
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700570/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 * del_timer_sync - deactivate a timer and wait for the handler to finish.
572 * @timer: the timer to be deactivated
573 *
574 * This function only differs from del_timer() on SMP: besides deactivating
575 * the timer it also makes sure the handler has finished executing on other
576 * CPUs.
577 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800578 * Synchronization rules: Callers must prevent restarting of the timer,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 * otherwise this function is meaningless. It must not be called from
580 * interrupt contexts. The caller must not hold locks which would prevent
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700581 * completion of the timer's handler. The timer's handler must not call
582 * add_timer_on(). Upon exit the timer is not queued and the handler is
583 * not running on any CPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 *
585 * The function returns whether it has deactivated a pending timer or not.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 */
587int del_timer_sync(struct timer_list *timer)
588{
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700589 for (;;) {
590 int ret = try_to_del_timer_sync(timer);
591 if (ret >= 0)
592 return ret;
Andrew Mortona0009652006-07-14 00:24:06 -0700593 cpu_relax();
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700594 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595}
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700596
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597EXPORT_SYMBOL(del_timer_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598#endif
599
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100600static int cascade(struct tvec_base *base, struct tvec *tv, int index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601{
602 /* cascade all the timers from tv up one level */
Porpoise3439dd82006-06-23 02:05:56 -0700603 struct timer_list *timer, *tmp;
604 struct list_head tv_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
Porpoise3439dd82006-06-23 02:05:56 -0700606 list_replace_init(tv->vec + index, &tv_list);
607
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 /*
Porpoise3439dd82006-06-23 02:05:56 -0700609 * We are removing _all_ timers from the list, so we
610 * don't have to detach them individually.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 */
Porpoise3439dd82006-06-23 02:05:56 -0700612 list_for_each_entry_safe(timer, tmp, &tv_list, entry) {
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700613 BUG_ON(tbase_get_base(timer->base) != base);
Porpoise3439dd82006-06-23 02:05:56 -0700614 internal_add_timer(base, timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
617 return index;
618}
619
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700620#define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
621
622/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 * __run_timers - run all expired timers (if any) on this CPU.
624 * @base: the timer vector to be processed.
625 *
626 * This function cascades all vectors and executes all expired timer
627 * vectors.
628 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100629static inline void __run_timers(struct tvec_base *base)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630{
631 struct timer_list *timer;
632
Oleg Nesterov3691c512006-03-31 02:30:30 -0800633 spin_lock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 while (time_after_eq(jiffies, base->timer_jiffies)) {
Oleg Nesterov626ab0e2006-06-23 02:05:55 -0700635 struct list_head work_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 struct list_head *head = &work_list;
Thomas Gleixner68194572007-07-19 01:49:16 -0700637 int index = base->timer_jiffies & TVR_MASK;
Oleg Nesterov626ab0e2006-06-23 02:05:55 -0700638
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 /*
640 * Cascade timers:
641 */
642 if (!index &&
643 (!cascade(base, &base->tv2, INDEX(0))) &&
644 (!cascade(base, &base->tv3, INDEX(1))) &&
645 !cascade(base, &base->tv4, INDEX(2)))
646 cascade(base, &base->tv5, INDEX(3));
Oleg Nesterov626ab0e2006-06-23 02:05:55 -0700647 ++base->timer_jiffies;
648 list_replace_init(base->tv1.vec + index, &work_list);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700649 while (!list_empty(head)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 void (*fn)(unsigned long);
651 unsigned long data;
652
Pavel Emelianovb5e61812007-05-08 00:30:19 -0700653 timer = list_first_entry(head, struct timer_list,entry);
Thomas Gleixner68194572007-07-19 01:49:16 -0700654 fn = timer->function;
655 data = timer->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800657 timer_stats_account_timer(timer);
658
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 set_running_timer(base, timer);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700660 detach_timer(timer, 1);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800661 spin_unlock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 {
Jesper Juhlbe5b4fb2005-06-23 00:09:09 -0700663 int preempt_count = preempt_count();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 fn(data);
665 if (preempt_count != preempt_count()) {
Pavel Machek4c9dc642008-01-30 13:30:00 +0100666 printk(KERN_ERR "huh, entered %p "
Jesper Juhlbe5b4fb2005-06-23 00:09:09 -0700667 "with preempt_count %08x, exited"
668 " with %08x?\n",
669 fn, preempt_count,
670 preempt_count());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 BUG();
672 }
673 }
Oleg Nesterov3691c512006-03-31 02:30:30 -0800674 spin_lock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 }
676 }
677 set_running_timer(base, NULL);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800678 spin_unlock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679}
680
Thomas Gleixnerfd064b92007-02-16 01:27:47 -0800681#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682/*
683 * Find out when the next timer event is due to happen. This
684 * is used on S/390 to stop all activity when a cpus is idle.
685 * This functions needs to be called disabled.
686 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100687static unsigned long __next_timer_interrupt(struct tvec_base *base)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688{
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800689 unsigned long timer_jiffies = base->timer_jiffies;
Thomas Gleixnereaad0842007-05-29 23:47:39 +0200690 unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800691 int index, slot, array, found = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 struct timer_list *nte;
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100693 struct tvec *varray[4];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694
695 /* Look for timer events in tv1. */
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800696 index = slot = timer_jiffies & TVR_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697 do {
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800698 list_for_each_entry(nte, base->tv1.vec + slot, entry) {
Thomas Gleixner68194572007-07-19 01:49:16 -0700699 if (tbase_get_deferrable(nte->base))
700 continue;
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700701
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800702 found = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 expires = nte->expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800704 /* Look at the cascade bucket(s)? */
705 if (!index || slot < index)
706 goto cascade;
707 return expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 }
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800709 slot = (slot + 1) & TVR_MASK;
710 } while (slot != index);
711
712cascade:
713 /* Calculate the next cascade event */
714 if (index)
715 timer_jiffies += TVR_SIZE - index;
716 timer_jiffies >>= TVR_BITS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717
718 /* Check tv2-tv5. */
719 varray[0] = &base->tv2;
720 varray[1] = &base->tv3;
721 varray[2] = &base->tv4;
722 varray[3] = &base->tv5;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800723
724 for (array = 0; array < 4; array++) {
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100725 struct tvec *varp = varray[array];
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800726
727 index = slot = timer_jiffies & TVN_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 do {
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800729 list_for_each_entry(nte, varp->vec + slot, entry) {
730 found = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 if (time_before(nte->expires, expires))
732 expires = nte->expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800733 }
734 /*
735 * Do we still search for the first timer or are
736 * we looking up the cascade buckets ?
737 */
738 if (found) {
739 /* Look at the cascade bucket(s)? */
740 if (!index || slot < index)
741 break;
742 return expires;
743 }
744 slot = (slot + 1) & TVN_MASK;
745 } while (slot != index);
746
747 if (index)
748 timer_jiffies += TVN_SIZE - index;
749 timer_jiffies >>= TVN_BITS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 }
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800751 return expires;
752}
753
754/*
755 * Check, if the next hrtimer event is before the next timer wheel
756 * event:
757 */
758static unsigned long cmp_next_hrtimer_event(unsigned long now,
759 unsigned long expires)
760{
761 ktime_t hr_delta = hrtimer_get_next_event();
762 struct timespec tsdelta;
Thomas Gleixner9501b6c2007-03-25 14:31:17 +0200763 unsigned long delta;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800764
765 if (hr_delta.tv64 == KTIME_MAX)
766 return expires;
767
Thomas Gleixner9501b6c2007-03-25 14:31:17 +0200768 /*
769 * Expired timer available, let it expire in the next tick
770 */
771 if (hr_delta.tv64 <= 0)
772 return now + 1;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800773
774 tsdelta = ktime_to_timespec(hr_delta);
Thomas Gleixner9501b6c2007-03-25 14:31:17 +0200775 delta = timespec_to_jiffies(&tsdelta);
Thomas Gleixnereaad0842007-05-29 23:47:39 +0200776
777 /*
778 * Limit the delta to the max value, which is checked in
779 * tick_nohz_stop_sched_tick():
780 */
781 if (delta > NEXT_TIMER_MAX_DELTA)
782 delta = NEXT_TIMER_MAX_DELTA;
783
Thomas Gleixner9501b6c2007-03-25 14:31:17 +0200784 /*
785 * Take rounding errors in to account and make sure, that it
786 * expires in the next tick. Otherwise we go into an endless
787 * ping pong due to tick_nohz_stop_sched_tick() retriggering
788 * the timer softirq
789 */
790 if (delta < 1)
791 delta = 1;
792 now += delta;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800793 if (time_before(now, expires))
794 return now;
795 return expires;
796}
797
798/**
Li Zefan8dce39c2007-11-05 14:51:10 -0800799 * get_next_timer_interrupt - return the jiffy of the next pending timer
Randy Dunlap05fb6bf2007-02-28 20:12:13 -0800800 * @now: current time (in jiffies)
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800801 */
Thomas Gleixnerfd064b92007-02-16 01:27:47 -0800802unsigned long get_next_timer_interrupt(unsigned long now)
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800803{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100804 struct tvec_base *base = __get_cpu_var(tvec_bases);
Thomas Gleixnerfd064b92007-02-16 01:27:47 -0800805 unsigned long expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800806
807 spin_lock(&base->lock);
808 expires = __next_timer_interrupt(base);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800809 spin_unlock(&base->lock);
Tony Lindgren69239742006-03-06 15:42:45 -0800810
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800811 if (time_before_eq(expires, now))
812 return now;
Zachary Amsden0662b712006-05-20 15:00:24 -0700813
Thomas Gleixner1cfd6842007-02-16 01:27:46 -0800814 return cmp_next_hrtimer_event(now, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815}
Thomas Gleixnerfd064b92007-02-16 01:27:47 -0800816
817#ifdef CONFIG_NO_IDLE_HZ
818unsigned long next_timer_interrupt(void)
819{
820 return get_next_timer_interrupt(jiffies);
821}
822#endif
823
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824#endif
825
Paul Mackerrasfa13a5a2007-11-09 22:39:38 +0100826#ifndef CONFIG_VIRT_CPU_ACCOUNTING
827void account_process_tick(struct task_struct *p, int user_tick)
828{
Michael Neuling06b8e872008-02-06 01:36:12 -0800829 cputime_t one_jiffy = jiffies_to_cputime(1);
830
Paul Mackerrasfa13a5a2007-11-09 22:39:38 +0100831 if (user_tick) {
Michael Neuling06b8e872008-02-06 01:36:12 -0800832 account_user_time(p, one_jiffy);
833 account_user_time_scaled(p, cputime_to_scaled(one_jiffy));
Paul Mackerrasfa13a5a2007-11-09 22:39:38 +0100834 } else {
Michael Neuling06b8e872008-02-06 01:36:12 -0800835 account_system_time(p, HARDIRQ_OFFSET, one_jiffy);
836 account_system_time_scaled(p, cputime_to_scaled(one_jiffy));
Paul Mackerrasfa13a5a2007-11-09 22:39:38 +0100837 }
838}
839#endif
840
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841/*
Daniel Walker5b4db0c2007-10-18 03:06:11 -0700842 * Called from the timer interrupt handler to charge one tick to the current
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843 * process. user_tick is 1 if the tick is user time, 0 for system.
844 */
845void update_process_times(int user_tick)
846{
847 struct task_struct *p = current;
848 int cpu = smp_processor_id();
849
850 /* Note: this timer irq context must be accounted for as well. */
Paul Mackerrasfa13a5a2007-11-09 22:39:38 +0100851 account_process_tick(p, user_tick);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852 run_local_timers();
853 if (rcu_pending(cpu))
854 rcu_check_callbacks(cpu, user_tick);
855 scheduler_tick();
Thomas Gleixner68194572007-07-19 01:49:16 -0700856 run_posix_cpu_timers(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857}
858
859/*
860 * Nr of active tasks - counted in fixed-point numbers
861 */
862static unsigned long count_active_tasks(void)
863{
Jack Steinerdb1b1fe2006-03-31 02:31:21 -0800864 return nr_active() * FIXED_1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865}
866
867/*
868 * Hmm.. Changed this, as the GNU make sources (load.c) seems to
869 * imply that avenrun[] is the standard name for this kind of thing.
870 * Nothing else seems to be standardized: the fractional size etc
871 * all seem to differ on different machines.
872 *
873 * Requires xtime_lock to access.
874 */
875unsigned long avenrun[3];
876
877EXPORT_SYMBOL(avenrun);
878
879/*
880 * calc_load - given tick count, update the avenrun load estimates.
881 * This is called while holding a write_lock on xtime_lock.
882 */
883static inline void calc_load(unsigned long ticks)
884{
885 unsigned long active_tasks; /* fixed-point */
886 static int count = LOAD_FREQ;
887
Eric Dumazetcd7175e2006-12-13 00:35:45 -0800888 count -= ticks;
889 if (unlikely(count < 0)) {
890 active_tasks = count_active_tasks();
891 do {
892 CALC_LOAD(avenrun[0], EXP_1, active_tasks);
893 CALC_LOAD(avenrun[1], EXP_5, active_tasks);
894 CALC_LOAD(avenrun[2], EXP_15, active_tasks);
895 count += LOAD_FREQ;
896 } while (count < 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 }
898}
899
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 * This function runs timers and the timer-tq in bottom half context.
902 */
903static void run_timer_softirq(struct softirq_action *h)
904{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100905 struct tvec_base *base = __get_cpu_var(tvec_bases);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906
Peter Zijlstrad3d74452008-01-25 21:08:31 +0100907 hrtimer_run_pending();
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800908
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 if (time_after_eq(jiffies, base->timer_jiffies))
910 __run_timers(base);
911}
912
913/*
914 * Called by the local, per-CPU timer interrupt on SMP.
915 */
916void run_local_timers(void)
917{
Peter Zijlstrad3d74452008-01-25 21:08:31 +0100918 hrtimer_run_queues();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919 raise_softirq(TIMER_SOFTIRQ);
Ingo Molnar6687a972006-03-24 03:18:41 -0800920 softlockup_tick();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921}
922
923/*
924 * Called by the timer interrupt. xtime_lock must already be taken
925 * by the timer IRQ!
926 */
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700927static inline void update_times(unsigned long ticks)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928{
john stultzad596172006-06-26 00:25:06 -0700929 update_wall_time();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930 calc_load(ticks);
931}
Thomas Gleixner68194572007-07-19 01:49:16 -0700932
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933/*
934 * The 64-bit jiffies value is not atomic - you MUST NOT read it
935 * without sampling the sequence number in xtime_lock.
936 * jiffies is defined in the linker script...
937 */
938
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700939void do_timer(unsigned long ticks)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940{
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700941 jiffies_64 += ticks;
942 update_times(ticks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943}
944
945#ifdef __ARCH_WANT_SYS_ALARM
946
947/*
948 * For backwards compatibility? This can be done in libc so Alpha
949 * and all newer ports shouldn't need it.
950 */
951asmlinkage unsigned long sys_alarm(unsigned int seconds)
952{
Thomas Gleixnerc08b8a42006-03-25 03:06:33 -0800953 return alarm_setitimer(seconds);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954}
955
956#endif
957
958#ifndef __alpha__
959
960/*
961 * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this
962 * should be moved into arch/i386 instead?
963 */
964
965/**
966 * sys_getpid - return the thread group id of the current process
967 *
968 * Note, despite the name, this returns the tgid not the pid. The tgid and
969 * the pid are identical unless CLONE_THREAD was specified on clone() in
970 * which case the tgid is the same in all threads of the same group.
971 *
972 * This is SMP safe as current->tgid does not change.
973 */
974asmlinkage long sys_getpid(void)
975{
Pavel Emelyanovb4888932007-10-18 23:40:14 -0700976 return task_tgid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977}
978
979/*
Kirill Korotaev6997a6f2006-08-13 23:24:23 -0700980 * Accessing ->real_parent is not SMP-safe, it could
981 * change from under us. However, we can use a stale
982 * value of ->real_parent under rcu_read_lock(), see
983 * release_task()->call_rcu(delayed_put_task_struct).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984 */
985asmlinkage long sys_getppid(void)
986{
987 int pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988
Kirill Korotaev6997a6f2006-08-13 23:24:23 -0700989 rcu_read_lock();
Pavel Emelyanov6c5f3e72008-02-08 04:19:20 -0800990 pid = task_tgid_vnr(current->real_parent);
Kirill Korotaev6997a6f2006-08-13 23:24:23 -0700991 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 return pid;
994}
995
996asmlinkage long sys_getuid(void)
997{
998 /* Only we change this so SMP safe */
999 return current->uid;
1000}
1001
1002asmlinkage long sys_geteuid(void)
1003{
1004 /* Only we change this so SMP safe */
1005 return current->euid;
1006}
1007
1008asmlinkage long sys_getgid(void)
1009{
1010 /* Only we change this so SMP safe */
1011 return current->gid;
1012}
1013
1014asmlinkage long sys_getegid(void)
1015{
1016 /* Only we change this so SMP safe */
1017 return current->egid;
1018}
1019
1020#endif
1021
1022static void process_timeout(unsigned long __data)
1023{
Ingo Molnar36c8b582006-07-03 00:25:41 -07001024 wake_up_process((struct task_struct *)__data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025}
1026
1027/**
1028 * schedule_timeout - sleep until timeout
1029 * @timeout: timeout value in jiffies
1030 *
1031 * Make the current task sleep until @timeout jiffies have
1032 * elapsed. The routine will return immediately unless
1033 * the current task state has been set (see set_current_state()).
1034 *
1035 * You can set the task state as follows -
1036 *
1037 * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
1038 * pass before the routine returns. The routine will return 0
1039 *
1040 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1041 * delivered to the current task. In this case the remaining time
1042 * in jiffies will be returned, or 0 if the timer expired in time
1043 *
1044 * The current task state is guaranteed to be TASK_RUNNING when this
1045 * routine returns.
1046 *
1047 * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
1048 * the CPU away without a bound on the timeout. In this case the return
1049 * value will be %MAX_SCHEDULE_TIMEOUT.
1050 *
1051 * In all cases the return value is guaranteed to be non-negative.
1052 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08001053signed long __sched schedule_timeout(signed long timeout)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054{
1055 struct timer_list timer;
1056 unsigned long expire;
1057
1058 switch (timeout)
1059 {
1060 case MAX_SCHEDULE_TIMEOUT:
1061 /*
1062 * These two special cases are useful to be comfortable
1063 * in the caller. Nothing more. We could take
1064 * MAX_SCHEDULE_TIMEOUT from one of the negative value
1065 * but I' d like to return a valid offset (>=0) to allow
1066 * the caller to do everything it want with the retval.
1067 */
1068 schedule();
1069 goto out;
1070 default:
1071 /*
1072 * Another bit of PARANOID. Note that the retval will be
1073 * 0 since no piece of kernel is supposed to do a check
1074 * for a negative retval of schedule_timeout() (since it
1075 * should never happens anyway). You just have the printk()
1076 * that will tell you if something is gone wrong and where.
1077 */
Andrew Morton5b149bc2006-12-22 01:10:14 -08001078 if (timeout < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 printk(KERN_ERR "schedule_timeout: wrong timeout "
Andrew Morton5b149bc2006-12-22 01:10:14 -08001080 "value %lx\n", timeout);
1081 dump_stack();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082 current->state = TASK_RUNNING;
1083 goto out;
1084 }
1085 }
1086
1087 expire = timeout + jiffies;
1088
Oleg Nesterova8db2db2005-10-30 15:01:38 -08001089 setup_timer(&timer, process_timeout, (unsigned long)current);
1090 __mod_timer(&timer, expire);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091 schedule();
1092 del_singleshot_timer_sync(&timer);
1093
1094 timeout = expire - jiffies;
1095
1096 out:
1097 return timeout < 0 ? 0 : timeout;
1098}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099EXPORT_SYMBOL(schedule_timeout);
1100
Andrew Morton8a1c1752005-09-13 01:25:15 -07001101/*
1102 * We can use __set_current_state() here because schedule_timeout() calls
1103 * schedule() unconditionally.
1104 */
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001105signed long __sched schedule_timeout_interruptible(signed long timeout)
1106{
Andrew Mortona5a0d522005-10-30 15:01:42 -08001107 __set_current_state(TASK_INTERRUPTIBLE);
1108 return schedule_timeout(timeout);
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001109}
1110EXPORT_SYMBOL(schedule_timeout_interruptible);
1111
Matthew Wilcox294d5cc2007-12-06 11:59:46 -05001112signed long __sched schedule_timeout_killable(signed long timeout)
1113{
1114 __set_current_state(TASK_KILLABLE);
1115 return schedule_timeout(timeout);
1116}
1117EXPORT_SYMBOL(schedule_timeout_killable);
1118
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001119signed long __sched schedule_timeout_uninterruptible(signed long timeout)
1120{
Andrew Mortona5a0d522005-10-30 15:01:42 -08001121 __set_current_state(TASK_UNINTERRUPTIBLE);
1122 return schedule_timeout(timeout);
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001123}
1124EXPORT_SYMBOL(schedule_timeout_uninterruptible);
1125
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126/* Thread ID - the internal kernel "pid" */
1127asmlinkage long sys_gettid(void)
1128{
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001129 return task_pid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130}
1131
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -07001132/**
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001133 * do_sysinfo - fill in sysinfo struct
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -07001134 * @info: pointer to buffer to fill
Thomas Gleixner68194572007-07-19 01:49:16 -07001135 */
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001136int do_sysinfo(struct sysinfo *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 unsigned long mem_total, sav_total;
1139 unsigned int mem_unit, bitcount;
1140 unsigned long seq;
1141
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001142 memset(info, 0, sizeof(struct sysinfo));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
1144 do {
1145 struct timespec tp;
1146 seq = read_seqbegin(&xtime_lock);
1147
1148 /*
1149 * This is annoying. The below is the same thing
1150 * posix_get_clock_monotonic() does, but it wants to
1151 * take the lock which we want to cover the loads stuff
1152 * too.
1153 */
1154
1155 getnstimeofday(&tp);
1156 tp.tv_sec += wall_to_monotonic.tv_sec;
1157 tp.tv_nsec += wall_to_monotonic.tv_nsec;
Tomas Janousekd6214142007-07-15 23:39:42 -07001158 monotonic_to_bootbased(&tp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159 if (tp.tv_nsec - NSEC_PER_SEC >= 0) {
1160 tp.tv_nsec = tp.tv_nsec - NSEC_PER_SEC;
1161 tp.tv_sec++;
1162 }
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001163 info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001165 info->loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT);
1166 info->loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT);
1167 info->loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001169 info->procs = nr_threads;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170 } while (read_seqretry(&xtime_lock, seq));
1171
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001172 si_meminfo(info);
1173 si_swapinfo(info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174
1175 /*
1176 * If the sum of all the available memory (i.e. ram + swap)
1177 * is less than can be stored in a 32 bit unsigned long then
1178 * we can be binary compatible with 2.2.x kernels. If not,
1179 * well, in that case 2.2.x was broken anyways...
1180 *
1181 * -Erik Andersen <andersee@debian.org>
1182 */
1183
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001184 mem_total = info->totalram + info->totalswap;
1185 if (mem_total < info->totalram || mem_total < info->totalswap)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186 goto out;
1187 bitcount = 0;
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001188 mem_unit = info->mem_unit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 while (mem_unit > 1) {
1190 bitcount++;
1191 mem_unit >>= 1;
1192 sav_total = mem_total;
1193 mem_total <<= 1;
1194 if (mem_total < sav_total)
1195 goto out;
1196 }
1197
1198 /*
1199 * If mem_total did not overflow, multiply all memory values by
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001200 * info->mem_unit and set it to 1. This leaves things compatible
Linus Torvalds1da177e2005-04-16 15:20:36 -07001201 * with 2.2.x, and also retains compatibility with earlier 2.4.x
1202 * kernels...
1203 */
1204
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001205 info->mem_unit = 1;
1206 info->totalram <<= bitcount;
1207 info->freeram <<= bitcount;
1208 info->sharedram <<= bitcount;
1209 info->bufferram <<= bitcount;
1210 info->totalswap <<= bitcount;
1211 info->freeswap <<= bitcount;
1212 info->totalhigh <<= bitcount;
1213 info->freehigh <<= bitcount;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001215out:
1216 return 0;
1217}
1218
1219asmlinkage long sys_sysinfo(struct sysinfo __user *info)
1220{
1221 struct sysinfo val;
1222
1223 do_sysinfo(&val);
1224
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225 if (copy_to_user(info, &val, sizeof(struct sysinfo)))
1226 return -EFAULT;
1227
1228 return 0;
1229}
1230
Ingo Molnard730e882006-07-03 00:25:10 -07001231/*
1232 * lockdep: we want to track each per-CPU base as a separate lock-class,
1233 * but timer-bases are kmalloc()-ed, so we need to attach separate
1234 * keys to them:
1235 */
1236static struct lock_class_key base_lock_keys[NR_CPUS];
1237
Adrian Bunkb4be6252007-12-18 18:05:58 +01001238static int __cpuinit init_timers_cpu(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239{
1240 int j;
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001241 struct tvec_base *base;
Adrian Bunkb4be6252007-12-18 18:05:58 +01001242 static char __cpuinitdata tvec_base_done[NR_CPUS];
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001243
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001244 if (!tvec_base_done[cpu]) {
Jan Beulicha4a61982006-03-24 03:15:54 -08001245 static char boot_done;
1246
Jan Beulicha4a61982006-03-24 03:15:54 -08001247 if (boot_done) {
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001248 /*
1249 * The APs use this path later in boot
1250 */
Christoph Lameter94f60302007-07-17 04:03:29 -07001251 base = kmalloc_node(sizeof(*base),
1252 GFP_KERNEL | __GFP_ZERO,
Jan Beulicha4a61982006-03-24 03:15:54 -08001253 cpu_to_node(cpu));
1254 if (!base)
1255 return -ENOMEM;
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001256
1257 /* Make sure that tvec_base is 2 byte aligned */
1258 if (tbase_get_deferrable(base)) {
1259 WARN_ON(1);
1260 kfree(base);
1261 return -ENOMEM;
1262 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001263 per_cpu(tvec_bases, cpu) = base;
Jan Beulicha4a61982006-03-24 03:15:54 -08001264 } else {
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001265 /*
1266 * This is for the boot CPU - we use compile-time
1267 * static initialisation because per-cpu memory isn't
1268 * ready yet and because the memory allocators are not
1269 * initialised either.
1270 */
Jan Beulicha4a61982006-03-24 03:15:54 -08001271 boot_done = 1;
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001272 base = &boot_tvec_bases;
Jan Beulicha4a61982006-03-24 03:15:54 -08001273 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001274 tvec_base_done[cpu] = 1;
1275 } else {
1276 base = per_cpu(tvec_bases, cpu);
Jan Beulicha4a61982006-03-24 03:15:54 -08001277 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001278
Oleg Nesterov3691c512006-03-31 02:30:30 -08001279 spin_lock_init(&base->lock);
Ingo Molnard730e882006-07-03 00:25:10 -07001280 lockdep_set_class(&base->lock, base_lock_keys + cpu);
1281
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 for (j = 0; j < TVN_SIZE; j++) {
1283 INIT_LIST_HEAD(base->tv5.vec + j);
1284 INIT_LIST_HEAD(base->tv4.vec + j);
1285 INIT_LIST_HEAD(base->tv3.vec + j);
1286 INIT_LIST_HEAD(base->tv2.vec + j);
1287 }
1288 for (j = 0; j < TVR_SIZE; j++)
1289 INIT_LIST_HEAD(base->tv1.vec + j);
1290
1291 base->timer_jiffies = jiffies;
Jan Beulicha4a61982006-03-24 03:15:54 -08001292 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293}
1294
1295#ifdef CONFIG_HOTPLUG_CPU
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001296static void migrate_timer_list(struct tvec_base *new_base, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297{
1298 struct timer_list *timer;
1299
1300 while (!list_empty(head)) {
Pavel Emelianovb5e61812007-05-08 00:30:19 -07001301 timer = list_first_entry(head, struct timer_list, entry);
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001302 detach_timer(timer, 0);
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001303 timer_set_base(timer, new_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304 internal_add_timer(new_base, timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306}
1307
Randy Dunlap48ccf3d2008-01-21 17:18:25 -08001308static void __cpuinit migrate_timers(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309{
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001310 struct tvec_base *old_base;
1311 struct tvec_base *new_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312 int i;
1313
1314 BUG_ON(cpu_online(cpu));
Jan Beulicha4a61982006-03-24 03:15:54 -08001315 old_base = per_cpu(tvec_bases, cpu);
1316 new_base = get_cpu_var(tvec_bases);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317
1318 local_irq_disable();
Heiko Carstense81ce1f2007-03-05 00:30:51 -08001319 double_spin_lock(&new_base->lock, &old_base->lock,
1320 smp_processor_id() < cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321
Oleg Nesterov3691c512006-03-31 02:30:30 -08001322 BUG_ON(old_base->running_timer);
1323
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 for (i = 0; i < TVR_SIZE; i++)
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001325 migrate_timer_list(new_base, old_base->tv1.vec + i);
1326 for (i = 0; i < TVN_SIZE; i++) {
1327 migrate_timer_list(new_base, old_base->tv2.vec + i);
1328 migrate_timer_list(new_base, old_base->tv3.vec + i);
1329 migrate_timer_list(new_base, old_base->tv4.vec + i);
1330 migrate_timer_list(new_base, old_base->tv5.vec + i);
1331 }
1332
Heiko Carstense81ce1f2007-03-05 00:30:51 -08001333 double_spin_unlock(&new_base->lock, &old_base->lock,
1334 smp_processor_id() < cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 local_irq_enable();
1336 put_cpu_var(tvec_bases);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337}
1338#endif /* CONFIG_HOTPLUG_CPU */
1339
Chandra Seetharaman8c78f302006-07-30 03:03:35 -07001340static int __cpuinit timer_cpu_notify(struct notifier_block *self,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341 unsigned long action, void *hcpu)
1342{
1343 long cpu = (long)hcpu;
1344 switch(action) {
1345 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001346 case CPU_UP_PREPARE_FROZEN:
Jan Beulicha4a61982006-03-24 03:15:54 -08001347 if (init_timers_cpu(cpu) < 0)
1348 return NOTIFY_BAD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349 break;
1350#ifdef CONFIG_HOTPLUG_CPU
1351 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001352 case CPU_DEAD_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353 migrate_timers(cpu);
1354 break;
1355#endif
1356 default:
1357 break;
1358 }
1359 return NOTIFY_OK;
1360}
1361
Chandra Seetharaman8c78f302006-07-30 03:03:35 -07001362static struct notifier_block __cpuinitdata timers_nb = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 .notifier_call = timer_cpu_notify,
1364};
1365
1366
1367void __init init_timers(void)
1368{
Akinobu Mita07dccf32006-09-29 02:00:22 -07001369 int err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370 (void *)(long)smp_processor_id());
Akinobu Mita07dccf32006-09-29 02:00:22 -07001371
Ingo Molnar82f67cd2007-02-16 01:28:13 -08001372 init_timer_stats();
1373
Akinobu Mita07dccf32006-09-29 02:00:22 -07001374 BUG_ON(err == NOTIFY_BAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375 register_cpu_notifier(&timers_nb);
1376 open_softirq(TIMER_SOFTIRQ, run_timer_softirq, NULL);
1377}
1378
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379/**
1380 * msleep - sleep safely even with waitqueue interruptions
1381 * @msecs: Time in milliseconds to sleep for
1382 */
1383void msleep(unsigned int msecs)
1384{
1385 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1386
Nishanth Aravamudan75bcc8c2005-09-10 00:27:24 -07001387 while (timeout)
1388 timeout = schedule_timeout_uninterruptible(timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389}
1390
1391EXPORT_SYMBOL(msleep);
1392
1393/**
Domen Puncer96ec3ef2005-06-25 14:58:43 -07001394 * msleep_interruptible - sleep waiting for signals
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395 * @msecs: Time in milliseconds to sleep for
1396 */
1397unsigned long msleep_interruptible(unsigned int msecs)
1398{
1399 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1400
Nishanth Aravamudan75bcc8c2005-09-10 00:27:24 -07001401 while (timeout && !signal_pending(current))
1402 timeout = schedule_timeout_interruptible(timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403 return jiffies_to_msecs(timeout);
1404}
1405
1406EXPORT_SYMBOL(msleep_interruptible);