Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * Virtual cpu timer based timer functions. |
| 3 | * |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 4 | * Copyright IBM Corp. 2004, 2012 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * Author(s): Jan Glauber <jan.glauber@de.ibm.com> |
| 6 | */ |
| 7 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | #include <linux/kernel_stat.h> |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 9 | #include <linux/notifier.h> |
Martin Schwidefsky | 860dba4 | 2011-01-05 12:47:25 +0100 | [diff] [blame] | 10 | #include <linux/kprobes.h> |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 11 | #include <linux/export.h> |
| 12 | #include <linux/kernel.h> |
| 13 | #include <linux/timex.h> |
| 14 | #include <linux/types.h> |
| 15 | #include <linux/time.h> |
| 16 | #include <linux/cpu.h> |
| 17 | #include <linux/smp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | |
Heiko Carstens | 5a489b9 | 2006-10-06 16:38:35 +0200 | [diff] [blame] | 19 | #include <asm/irq_regs.h> |
Martin Schwidefsky | 76d4e00 | 2009-06-12 10:26:21 +0200 | [diff] [blame] | 20 | #include <asm/cputime.h> |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 21 | #include <asm/vtimer.h> |
Heiko Carstens | d7b250e | 2011-05-26 09:48:24 +0200 | [diff] [blame] | 22 | #include <asm/irq.h> |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 23 | #include "entry.h" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 25 | static void virt_timer_expire(void); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 27 | DEFINE_PER_CPU(struct s390_idle_data, s390_idle); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 28 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 29 | static LIST_HEAD(virt_timer_list); |
| 30 | static DEFINE_SPINLOCK(virt_timer_lock); |
| 31 | static atomic64_t virt_timer_current; |
| 32 | static atomic64_t virt_timer_elapsed; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 33 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 34 | static inline u64 get_vtimer(void) |
| 35 | { |
| 36 | u64 timer; |
| 37 | |
| 38 | asm volatile("stpt %0" : "=m" (timer)); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 39 | return timer; |
| 40 | } |
| 41 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 42 | static inline void set_vtimer(u64 expires) |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 43 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 44 | u64 timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 45 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 46 | asm volatile( |
| 47 | " stpt %0\n" /* Store current cpu timer value */ |
| 48 | " spt %1" /* Set new value imm. afterwards */ |
| 49 | : "=m" (timer) : "m" (expires)); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 50 | S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; |
| 51 | S390_lowcore.last_update_timer = expires; |
| 52 | } |
| 53 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 54 | static inline int virt_timer_forward(u64 elapsed) |
| 55 | { |
| 56 | BUG_ON(!irqs_disabled()); |
| 57 | |
| 58 | if (list_empty(&virt_timer_list)) |
| 59 | return 0; |
| 60 | elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); |
| 61 | return elapsed >= atomic64_read(&virt_timer_current); |
| 62 | } |
| 63 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | /* |
| 65 | * Update process times based on virtual cpu times stored by entry.S |
| 66 | * to the lowcore fields user_timer, system_timer & steal_clock. |
| 67 | */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 68 | static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | { |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 70 | struct thread_info *ti = task_thread_info(tsk); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 71 | u64 timer, clock, user, system, steal; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 72 | |
| 73 | timer = S390_lowcore.last_update_timer; |
| 74 | clock = S390_lowcore.last_update_clock; |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 75 | asm volatile( |
| 76 | " stpt %0\n" /* Store current cpu timer value */ |
| 77 | " stck %1" /* Store current tod clock value */ |
| 78 | : "=m" (S390_lowcore.last_update_timer), |
| 79 | "=m" (S390_lowcore.last_update_clock)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 80 | S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 81 | S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 83 | user = S390_lowcore.user_timer - ti->user_timer; |
| 84 | S390_lowcore.steal_timer -= user; |
| 85 | ti->user_timer = S390_lowcore.user_timer; |
| 86 | account_user_time(tsk, user, user); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 87 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 88 | system = S390_lowcore.system_timer - ti->system_timer; |
| 89 | S390_lowcore.steal_timer -= system; |
| 90 | ti->system_timer = S390_lowcore.system_timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 91 | account_system_time(tsk, hardirq_offset, system, system); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 92 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 93 | steal = S390_lowcore.steal_timer; |
| 94 | if ((s64) steal > 0) { |
| 95 | S390_lowcore.steal_timer = 0; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 96 | account_steal_time(steal); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | } |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 98 | |
| 99 | return virt_timer_forward(user + system); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | } |
| 101 | |
Frederic Weisbecker | bf9fae9 | 2012-09-08 15:23:11 +0200 | [diff] [blame] | 102 | void vtime_task_switch(struct task_struct *prev) |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 103 | { |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 104 | struct thread_info *ti; |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 105 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 106 | do_account_vtime(prev, 0); |
| 107 | ti = task_thread_info(prev); |
| 108 | ti->user_timer = S390_lowcore.user_timer; |
| 109 | ti->system_timer = S390_lowcore.system_timer; |
Frederic Weisbecker | baa3604 | 2012-06-18 17:54:14 +0200 | [diff] [blame] | 110 | ti = task_thread_info(current); |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 111 | S390_lowcore.user_timer = ti->user_timer; |
| 112 | S390_lowcore.system_timer = ti->system_timer; |
| 113 | } |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 114 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 115 | void account_process_tick(struct task_struct *tsk, int user_tick) |
| 116 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 117 | if (do_account_vtime(tsk, HARDIRQ_OFFSET)) |
| 118 | virt_timer_expire(); |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 119 | } |
| 120 | |
| 121 | /* |
| 122 | * Update process times based on virtual cpu times stored by entry.S |
| 123 | * to the lowcore fields user_timer, system_timer & steal_clock. |
| 124 | */ |
Frederic Weisbecker | bf9fae9 | 2012-09-08 15:23:11 +0200 | [diff] [blame] | 125 | void vtime_account(struct task_struct *tsk) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | { |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 127 | struct thread_info *ti = task_thread_info(tsk); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 128 | u64 timer, system; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | |
| 130 | timer = S390_lowcore.last_update_timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 131 | S390_lowcore.last_update_timer = get_vtimer(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; |
| 133 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 134 | system = S390_lowcore.system_timer - ti->system_timer; |
| 135 | S390_lowcore.steal_timer -= system; |
| 136 | ti->system_timer = S390_lowcore.system_timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 137 | account_system_time(tsk, 0, system, system); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 138 | |
| 139 | virt_timer_forward(system); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 140 | } |
Frederic Weisbecker | bf9fae9 | 2012-09-08 15:23:11 +0200 | [diff] [blame] | 141 | EXPORT_SYMBOL_GPL(vtime_account); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | |
Frederic Weisbecker | 1111333 | 2012-10-24 18:05:51 +0200 | [diff] [blame^] | 143 | void __vtime_account_system(struct task_struct *tsk) |
| 144 | __attribute__((alias("vtime_account"))); |
| 145 | EXPORT_SYMBOL_GPL(__vtime_account_system); |
| 146 | |
Martin Schwidefsky | 860dba4 | 2011-01-05 12:47:25 +0100 | [diff] [blame] | 147 | void __kprobes vtime_stop_cpu(void) |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 148 | { |
| 149 | struct s390_idle_data *idle = &__get_cpu_var(s390_idle); |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 150 | unsigned long long idle_time; |
| 151 | unsigned long psw_mask; |
| 152 | |
| 153 | trace_hardirqs_on(); |
| 154 | /* Don't trace preempt off for idle. */ |
| 155 | stop_critical_timings(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 157 | /* Wait for external, I/O or machine check interrupt. */ |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 158 | psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT | |
| 159 | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; |
Martin Schwidefsky | 3c5d92a | 2009-09-29 14:25:16 +0200 | [diff] [blame] | 160 | idle->nohz_delay = 0; |
| 161 | |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 162 | /* Call the assembler magic in entry.S */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 163 | psw_idle(idle, psw_mask); |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 164 | |
| 165 | /* Reenable preemption tracer. */ |
| 166 | start_critical_timings(); |
| 167 | |
| 168 | /* Account time spent with enabled wait psw loaded as idle time. */ |
| 169 | idle->sequence++; |
| 170 | smp_wmb(); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 171 | idle_time = idle->clock_idle_exit - idle->clock_idle_enter; |
| 172 | idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 173 | idle->idle_time += idle_time; |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 174 | idle->idle_count++; |
| 175 | account_idle_time(idle_time); |
| 176 | smp_wmb(); |
| 177 | idle->sequence++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 178 | } |
| 179 | |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 180 | cputime64_t s390_get_idle_time(int cpu) |
| 181 | { |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 182 | struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); |
| 183 | unsigned long long now, idle_enter, idle_exit; |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 184 | unsigned int sequence; |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 185 | |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 186 | do { |
| 187 | now = get_clock(); |
| 188 | sequence = ACCESS_ONCE(idle->sequence); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 189 | idle_enter = ACCESS_ONCE(idle->clock_idle_enter); |
| 190 | idle_exit = ACCESS_ONCE(idle->clock_idle_exit); |
Martin Schwidefsky | 4c1051e | 2012-03-11 11:59:27 -0400 | [diff] [blame] | 191 | } while ((sequence & 1) || (idle->sequence != sequence)); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 192 | return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0; |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 193 | } |
| 194 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 195 | /* |
| 196 | * Sorted add to a list. List is linear searched until first bigger |
| 197 | * element is found. |
| 198 | */ |
| 199 | static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) |
| 200 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 201 | struct vtimer_list *tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 202 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 203 | list_for_each_entry(tmp, head, entry) { |
| 204 | if (tmp->expires > timer->expires) { |
| 205 | list_add_tail(&timer->entry, &tmp->entry); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | return; |
| 207 | } |
| 208 | } |
| 209 | list_add_tail(&timer->entry, head); |
| 210 | } |
| 211 | |
| 212 | /* |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 213 | * Handler for expired virtual CPU timer. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 214 | */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 215 | static void virt_timer_expire(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 216 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 217 | struct vtimer_list *timer, *tmp; |
| 218 | unsigned long elapsed; |
| 219 | LIST_HEAD(cb_list); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 221 | /* walk timer list, fire all expired timers */ |
| 222 | spin_lock(&virt_timer_lock); |
| 223 | elapsed = atomic64_read(&virt_timer_elapsed); |
| 224 | list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { |
| 225 | if (timer->expires < elapsed) |
| 226 | /* move expired timer to the callback queue */ |
| 227 | list_move_tail(&timer->entry, &cb_list); |
| 228 | else |
| 229 | timer->expires -= elapsed; |
| 230 | } |
| 231 | if (!list_empty(&virt_timer_list)) { |
| 232 | timer = list_first_entry(&virt_timer_list, |
| 233 | struct vtimer_list, entry); |
| 234 | atomic64_set(&virt_timer_current, timer->expires); |
| 235 | } |
| 236 | atomic64_sub(elapsed, &virt_timer_elapsed); |
| 237 | spin_unlock(&virt_timer_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 238 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 239 | /* Do callbacks and recharge periodic timers */ |
| 240 | list_for_each_entry_safe(timer, tmp, &cb_list, entry) { |
| 241 | list_del_init(&timer->entry); |
| 242 | timer->function(timer->data); |
| 243 | if (timer->interval) { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 244 | /* Recharge interval timer */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 245 | timer->expires = timer->interval + |
| 246 | atomic64_read(&virt_timer_elapsed); |
| 247 | spin_lock(&virt_timer_lock); |
| 248 | list_add_sorted(timer, &virt_timer_list); |
| 249 | spin_unlock(&virt_timer_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | } |
| 251 | } |
| 252 | } |
| 253 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 254 | void init_virt_timer(struct vtimer_list *timer) |
| 255 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 256 | timer->function = NULL; |
| 257 | INIT_LIST_HEAD(&timer->entry); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 258 | } |
| 259 | EXPORT_SYMBOL(init_virt_timer); |
| 260 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 261 | static inline int vtimer_pending(struct vtimer_list *timer) |
| 262 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 263 | return !list_empty(&timer->entry); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 264 | } |
| 265 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | static void internal_add_vtimer(struct vtimer_list *timer) |
| 267 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 268 | if (list_empty(&virt_timer_list)) { |
| 269 | /* First timer, just program it. */ |
| 270 | atomic64_set(&virt_timer_current, timer->expires); |
| 271 | atomic64_set(&virt_timer_elapsed, 0); |
| 272 | list_add(&timer->entry, &virt_timer_list); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 273 | } else { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 274 | /* Update timer against current base. */ |
| 275 | timer->expires += atomic64_read(&virt_timer_elapsed); |
| 276 | if (likely((s64) timer->expires < |
| 277 | (s64) atomic64_read(&virt_timer_current))) |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 278 | /* The new timer expires before the current timer. */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 279 | atomic64_set(&virt_timer_current, timer->expires); |
| 280 | /* Insert new timer into the list. */ |
| 281 | list_add_sorted(timer, &virt_timer_list); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 283 | } |
| 284 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 285 | static void __add_vtimer(struct vtimer_list *timer, int periodic) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 286 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 287 | unsigned long flags; |
| 288 | |
| 289 | timer->interval = periodic ? timer->expires : 0; |
| 290 | spin_lock_irqsave(&virt_timer_lock, flags); |
| 291 | internal_add_vtimer(timer); |
| 292 | spin_unlock_irqrestore(&virt_timer_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 293 | } |
| 294 | |
| 295 | /* |
| 296 | * add_virt_timer - add an oneshot virtual CPU timer |
| 297 | */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 298 | void add_virt_timer(struct vtimer_list *timer) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 300 | __add_vtimer(timer, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 301 | } |
| 302 | EXPORT_SYMBOL(add_virt_timer); |
| 303 | |
| 304 | /* |
| 305 | * add_virt_timer_int - add an interval virtual CPU timer |
| 306 | */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 307 | void add_virt_timer_periodic(struct vtimer_list *timer) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 308 | { |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 309 | __add_vtimer(timer, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 310 | } |
| 311 | EXPORT_SYMBOL(add_virt_timer_periodic); |
| 312 | |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 313 | static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 314 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 315 | unsigned long flags; |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 316 | int rc; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 317 | |
Martin Schwidefsky | ca366a3 | 2008-07-14 09:59:23 +0200 | [diff] [blame] | 318 | BUG_ON(!timer->function); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 319 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 320 | if (timer->expires == expires && vtimer_pending(timer)) |
| 321 | return 1; |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 322 | spin_lock_irqsave(&virt_timer_lock, flags); |
| 323 | rc = vtimer_pending(timer); |
| 324 | if (rc) |
| 325 | list_del_init(&timer->entry); |
| 326 | timer->interval = periodic ? expires : 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 327 | timer->expires = expires; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 328 | internal_add_vtimer(timer); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 329 | spin_unlock_irqrestore(&virt_timer_lock, flags); |
| 330 | return rc; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | } |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 332 | |
| 333 | /* |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 334 | * returns whether it has modified a pending timer (1) or not (0) |
| 335 | */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 336 | int mod_virt_timer(struct vtimer_list *timer, u64 expires) |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 337 | { |
| 338 | return __mod_vtimer(timer, expires, 0); |
| 339 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 340 | EXPORT_SYMBOL(mod_virt_timer); |
| 341 | |
| 342 | /* |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 343 | * returns whether it has modified a pending timer (1) or not (0) |
| 344 | */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 345 | int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 346 | { |
| 347 | return __mod_vtimer(timer, expires, 1); |
| 348 | } |
| 349 | EXPORT_SYMBOL(mod_virt_timer_periodic); |
| 350 | |
| 351 | /* |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 352 | * Delete a virtual timer. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 353 | * |
| 354 | * returns whether the deleted timer was pending (1) or not (0) |
| 355 | */ |
| 356 | int del_virt_timer(struct vtimer_list *timer) |
| 357 | { |
| 358 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 359 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 360 | if (!vtimer_pending(timer)) |
| 361 | return 0; |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 362 | spin_lock_irqsave(&virt_timer_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 363 | list_del_init(&timer->entry); |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 364 | spin_unlock_irqrestore(&virt_timer_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 365 | return 1; |
| 366 | } |
| 367 | EXPORT_SYMBOL(del_virt_timer); |
| 368 | |
| 369 | /* |
| 370 | * Start the virtual CPU timer on the current CPU. |
| 371 | */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 372 | void __cpuinit init_cpu_vtimer(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 373 | { |
Martin Schwidefsky | 8b646bd | 2012-03-11 11:59:26 -0400 | [diff] [blame] | 374 | /* set initial cpu timer */ |
Martin Schwidefsky | 27f6b41 | 2012-07-20 11:15:08 +0200 | [diff] [blame] | 375 | set_vtimer(VTIMER_MAX_SLICE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 376 | } |
| 377 | |
Heiko Carstens | 3988121 | 2010-12-01 10:08:01 +0100 | [diff] [blame] | 378 | static int __cpuinit s390_nohz_notify(struct notifier_block *self, |
| 379 | unsigned long action, void *hcpu) |
| 380 | { |
| 381 | struct s390_idle_data *idle; |
| 382 | long cpu = (long) hcpu; |
| 383 | |
| 384 | idle = &per_cpu(s390_idle, cpu); |
Heiko Carstens | 1c72592 | 2012-08-27 15:43:49 +0200 | [diff] [blame] | 385 | switch (action & ~CPU_TASKS_FROZEN) { |
Heiko Carstens | 3988121 | 2010-12-01 10:08:01 +0100 | [diff] [blame] | 386 | case CPU_DYING: |
Heiko Carstens | 3988121 | 2010-12-01 10:08:01 +0100 | [diff] [blame] | 387 | idle->nohz_delay = 0; |
| 388 | default: |
| 389 | break; |
| 390 | } |
| 391 | return NOTIFY_OK; |
| 392 | } |
| 393 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 394 | void __init vtime_init(void) |
| 395 | { |
Martin Schwidefsky | d54853e | 2007-02-05 21:18:19 +0100 | [diff] [blame] | 396 | /* Enable cpu timer interrupts on the boot cpu. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 397 | init_cpu_vtimer(); |
Heiko Carstens | 3988121 | 2010-12-01 10:08:01 +0100 | [diff] [blame] | 398 | cpu_notifier(s390_nohz_notify, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 399 | } |