blob: 8957810ad7d1e348dd6315242c3411a863745be6 [file] [log] [blame]
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001#ifndef _ASM_X86_PARAVIRT_H
2#define _ASM_X86_PARAVIRT_H
Rusty Russelld3561b72006-12-07 02:14:07 +01003/* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +02005
6#ifdef CONFIG_PARAVIRT
Jeremy Fitzhardinge54321d92009-02-11 10:20:05 -08007#include <asm/pgtable_types.h>
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01008#include <asm/asm.h>
Rusty Russelld3561b72006-12-07 02:14:07 +01009
Jeremy Fitzhardingeac5672f2009-04-14 14:29:44 -070010#include <asm/paravirt_types.h>
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -080011
Rusty Russelld3561b72006-12-07 02:14:07 +010012#ifndef __ASSEMBLY__
Paul Gortmaker187f1882011-11-23 20:12:59 -050013#include <linux/bug.h>
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020014#include <linux/types.h>
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +020015#include <linux/cpumask.h>
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +020016
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020017static inline int paravirt_enabled(void)
18{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070019 return pv_info.paravirt_enabled;
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020020}
Rusty Russelld3561b72006-12-07 02:14:07 +010021
H. Peter Anvinfaca6222008-01-30 13:31:02 +010022static inline void load_sp0(struct tss_struct *tss,
Rusty Russelld3561b72006-12-07 02:14:07 +010023 struct thread_struct *thread)
24{
H. Peter Anvinfaca6222008-01-30 13:31:02 +010025 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
Rusty Russelld3561b72006-12-07 02:14:07 +010026}
27
Rusty Russelld3561b72006-12-07 02:14:07 +010028/* The paravirtualized CPUID instruction. */
29static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
30 unsigned int *ecx, unsigned int *edx)
31{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070032 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
Rusty Russelld3561b72006-12-07 02:14:07 +010033}
34
35/*
36 * These special macros can be used to get or set a debugging register
37 */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020038static inline unsigned long paravirt_get_debugreg(int reg)
39{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070040 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020041}
42#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
43static inline void set_debugreg(unsigned long val, int reg)
44{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070045 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020046}
Rusty Russelld3561b72006-12-07 02:14:07 +010047
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020048static inline void clts(void)
49{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070050 PVOP_VCALL0(pv_cpu_ops.clts);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020051}
Rusty Russelld3561b72006-12-07 02:14:07 +010052
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020053static inline unsigned long read_cr0(void)
54{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070055 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020056}
Rusty Russelld3561b72006-12-07 02:14:07 +010057
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020058static inline void write_cr0(unsigned long x)
59{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070060 PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020061}
Rusty Russelld3561b72006-12-07 02:14:07 +010062
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020063static inline unsigned long read_cr2(void)
64{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070065 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020066}
Rusty Russelld3561b72006-12-07 02:14:07 +010067
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020068static inline void write_cr2(unsigned long x)
69{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070070 PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020071}
Rusty Russelld3561b72006-12-07 02:14:07 +010072
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020073static inline unsigned long read_cr3(void)
74{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070075 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020076}
77
78static inline void write_cr3(unsigned long x)
79{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070080 PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020081}
82
Andy Lutomirski1e02ce42014-10-24 15:58:08 -070083static inline unsigned long __read_cr4(void)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020084{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070085 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020086}
Andy Lutomirski1e02ce42014-10-24 15:58:08 -070087static inline unsigned long __read_cr4_safe(void)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020088{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070089 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020090}
91
Andy Lutomirski1e02ce42014-10-24 15:58:08 -070092static inline void __write_cr4(unsigned long x)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020093{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070094 PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +020095}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020096
Glauber de Oliveira Costa94ea03c2008-01-30 13:33:19 +010097#ifdef CONFIG_X86_64
Glauber de Oliveira Costa4c9890c2008-01-30 13:33:19 +010098static inline unsigned long read_cr8(void)
99{
100 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
101}
102
103static inline void write_cr8(unsigned long x)
104{
105 PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
106}
Glauber de Oliveira Costa94ea03c2008-01-30 13:33:19 +0100107#endif
Glauber de Oliveira Costa4c9890c2008-01-30 13:33:19 +0100108
David Howellsdf9ee292010-10-07 14:08:55 +0100109static inline void arch_safe_halt(void)
Rusty Russelld3561b72006-12-07 02:14:07 +0100110{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700111 PVOP_VCALL0(pv_irq_ops.safe_halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100112}
113
114static inline void halt(void)
115{
Cliff Wickmanc8217b82010-12-13 10:51:57 -0600116 PVOP_VCALL0(pv_irq_ops.halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100117}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200118
119static inline void wbinvd(void)
120{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700121 PVOP_VCALL0(pv_cpu_ops.wbinvd);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200122}
Rusty Russelld3561b72006-12-07 02:14:07 +0100123
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700124#define get_kernel_rpl() (pv_info.kernel_rpl)
Rusty Russelld3561b72006-12-07 02:14:07 +0100125
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200126static inline u64 paravirt_read_msr(unsigned msr, int *err)
127{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700128 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200129}
Borislav Petkov132ec922009-08-31 09:50:09 +0200130
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200131static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
132{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700133 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200134}
135
Rusty Russell90a0a062007-05-02 19:27:10 +0200136/* These should all do BUG_ON(_err), but our headers are too tangled. */
Joe Perches49cd7402008-03-23 01:03:00 -0700137#define rdmsr(msr, val1, val2) \
138do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200139 int _err; \
140 u64 _l = paravirt_read_msr(msr, &_err); \
141 val1 = (u32)_l; \
142 val2 = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700143} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100144
Joe Perches49cd7402008-03-23 01:03:00 -0700145#define wrmsr(msr, val1, val2) \
146do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200147 paravirt_write_msr(msr, val1, val2); \
Joe Perches49cd7402008-03-23 01:03:00 -0700148} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100149
Joe Perches49cd7402008-03-23 01:03:00 -0700150#define rdmsrl(msr, val) \
151do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200152 int _err; \
153 val = paravirt_read_msr(msr, &_err); \
Joe Perches49cd7402008-03-23 01:03:00 -0700154} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100155
Joe Perches49cd7402008-03-23 01:03:00 -0700156#define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
157#define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b)
Rusty Russelld3561b72006-12-07 02:14:07 +0100158
159/* rdmsr with exception handling */
Joe Perches49cd7402008-03-23 01:03:00 -0700160#define rdmsr_safe(msr, a, b) \
161({ \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200162 int _err; \
163 u64 _l = paravirt_read_msr(msr, &_err); \
164 (*a) = (u32)_l; \
165 (*b) = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700166 _err; \
167})
Rusty Russelld3561b72006-12-07 02:14:07 +0100168
Andi Kleen1de87bd2008-03-22 10:59:28 +0100169static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
170{
171 int err;
172
173 *p = paravirt_read_msr(msr, &err);
174 return err;
175}
Borislav Petkov177fed12009-08-31 09:50:10 +0200176
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200177static inline u64 paravirt_read_tsc(void)
178{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700179 return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200180}
Rusty Russelld3561b72006-12-07 02:14:07 +0100181
Joe Perches49cd7402008-03-23 01:03:00 -0700182#define rdtscl(low) \
183do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200184 u64 _l = paravirt_read_tsc(); \
185 low = (int)_l; \
Joe Perches49cd7402008-03-23 01:03:00 -0700186} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100187
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200188#define rdtscll(val) (val = paravirt_read_tsc())
Rusty Russelld3561b72006-12-07 02:14:07 +0100189
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700190static inline unsigned long long paravirt_sched_clock(void)
191{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700192 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700193}
Zachary Amsden6cb9a832007-03-05 00:30:35 -0800194
Ingo Molnarc5905af2012-02-24 08:31:31 +0100195struct static_key;
196extern struct static_key paravirt_steal_enabled;
197extern struct static_key paravirt_steal_rq_enabled;
Glauber Costa3c404b52011-07-11 15:28:15 -0400198
199static inline u64 paravirt_steal_clock(int cpu)
200{
201 return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
202}
203
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200204static inline unsigned long long paravirt_read_pmc(int counter)
205{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700206 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200207}
208
Joe Perches49cd7402008-03-23 01:03:00 -0700209#define rdpmc(counter, low, high) \
210do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200211 u64 _l = paravirt_read_pmc(counter); \
212 low = (u32)_l; \
213 high = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700214} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100215
Andi Kleen1ff4d582012-06-05 17:56:50 -0700216#define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter))
217
Glauber de Oliveira Costae5aaac42008-01-30 13:32:05 +0100218static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
219{
220 return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
221}
222
223#define rdtscp(low, high, aux) \
224do { \
225 int __aux; \
226 unsigned long __val = paravirt_rdtscp(&__aux); \
227 (low) = (u32)__val; \
228 (high) = (u32)(__val >> 32); \
229 (aux) = __aux; \
230} while (0)
231
232#define rdtscpll(val, aux) \
233do { \
234 unsigned long __aux; \
235 val = paravirt_rdtscp(&__aux); \
236 (aux) = __aux; \
237} while (0)
238
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700239static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
240{
241 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
242}
243
244static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
245{
246 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
247}
248
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200249static inline void load_TR_desc(void)
250{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700251 PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200252}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100253static inline void load_gdt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200254{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700255 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200256}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100257static inline void load_idt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200258{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700259 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200260}
261static inline void set_ldt(const void *addr, unsigned entries)
262{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700263 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200264}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100265static inline void store_idt(struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200266{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700267 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200268}
269static inline unsigned long paravirt_store_tr(void)
270{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700271 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200272}
273#define store_tr(tr) ((tr) = paravirt_store_tr())
274static inline void load_TLS(struct thread_struct *t, unsigned cpu)
275{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700276 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200277}
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100278
Jeremy Fitzhardinge9f9d4892008-06-25 00:19:32 -0400279#ifdef CONFIG_X86_64
280static inline void load_gs_index(unsigned int gs)
281{
282 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
283}
284#endif
285
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100286static inline void write_ldt_entry(struct desc_struct *dt, int entry,
287 const void *desc)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200288{
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100289 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200290}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100291
292static inline void write_gdt_entry(struct desc_struct *dt, int entry,
293 void *desc, int type)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200294{
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100295 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200296}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100297
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100298static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200299{
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100300 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200301}
302static inline void set_iopl_mask(unsigned mask)
303{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700304 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200305}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200306
Rusty Russelld3561b72006-12-07 02:14:07 +0100307/* The paravirtualized I/O functions */
Joe Perches49cd7402008-03-23 01:03:00 -0700308static inline void slow_down_io(void)
309{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700310 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100311#ifdef REALLY_SLOW_IO
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700312 pv_cpu_ops.io_delay();
313 pv_cpu_ops.io_delay();
314 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100315#endif
316}
317
Zachary Amsdenae5da272007-02-13 13:26:21 +0100318#ifdef CONFIG_SMP
319static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
320 unsigned long start_esp)
321{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700322 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
323 phys_apicid, start_eip, start_esp);
Zachary Amsdenae5da272007-02-13 13:26:21 +0100324}
325#endif
Rusty Russell13623d72006-12-07 02:14:08 +0100326
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200327static inline void paravirt_activate_mm(struct mm_struct *prev,
328 struct mm_struct *next)
329{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700330 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200331}
332
Dave Hansena1ea1c02014-11-18 10:23:49 -0800333static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
334 struct mm_struct *mm)
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200335{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700336 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200337}
338
Dave Hansena1ea1c02014-11-18 10:23:49 -0800339static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200340{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700341 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200342}
343
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200344static inline void __flush_tlb(void)
345{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700346 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200347}
348static inline void __flush_tlb_global(void)
349{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700350 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200351}
352static inline void __flush_tlb_single(unsigned long addr)
353{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700354 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200355}
Rusty Russellda181a82006-12-07 02:14:08 +0100356
Rusty Russell4595f962009-01-10 21:58:09 -0800357static inline void flush_tlb_others(const struct cpumask *cpumask,
358 struct mm_struct *mm,
Alex Shie7b52ff2012-06-28 09:02:17 +0800359 unsigned long start,
360 unsigned long end)
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200361{
Alex Shie7b52ff2012-06-28 09:02:17 +0800362 PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200363}
364
Jeremy Fitzhardingeeba00452008-06-25 00:19:12 -0400365static inline int paravirt_pgd_alloc(struct mm_struct *mm)
366{
367 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
368}
369
370static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
371{
372 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
373}
374
Eduardo Habkostf8639932008-07-30 18:32:27 -0300375static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200376{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700377 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200378}
Eduardo Habkostf8639932008-07-30 18:32:27 -0300379static inline void paravirt_release_pte(unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200380{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700381 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200382}
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100383
Eduardo Habkostf8639932008-07-30 18:32:27 -0300384static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200385{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700386 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200387}
388
Eduardo Habkostf8639932008-07-30 18:32:27 -0300389static inline void paravirt_release_pmd(unsigned long pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200390{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700391 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200392}
393
Eduardo Habkostf8639932008-07-30 18:32:27 -0300394static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -0700395{
396 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
397}
Eduardo Habkostf8639932008-07-30 18:32:27 -0300398static inline void paravirt_release_pud(unsigned long pfn)
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -0700399{
400 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
401}
402
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200403static inline void pte_update(struct mm_struct *mm, unsigned long addr,
404 pte_t *ptep)
405{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700406 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200407}
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800408static inline void pmd_update(struct mm_struct *mm, unsigned long addr,
409 pmd_t *pmdp)
410{
411 PVOP_VCALL3(pv_mmu_ops.pmd_update, mm, addr, pmdp);
412}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200413
414static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
415 pte_t *ptep)
416{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700417 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200418}
419
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800420static inline void pmd_update_defer(struct mm_struct *mm, unsigned long addr,
421 pmd_t *pmdp)
422{
423 PVOP_VCALL3(pv_mmu_ops.pmd_update_defer, mm, addr, pmdp);
424}
425
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100426static inline pte_t __pte(pteval_t val)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200427{
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100428 pteval_t ret;
429
430 if (sizeof(pteval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800431 ret = PVOP_CALLEE2(pteval_t,
432 pv_mmu_ops.make_pte,
433 val, (u64)val >> 32);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100434 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800435 ret = PVOP_CALLEE1(pteval_t,
436 pv_mmu_ops.make_pte,
437 val);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100438
Jeremy Fitzhardingec8e53932008-01-30 13:32:57 +0100439 return (pte_t) { .pte = ret };
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200440}
441
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100442static inline pteval_t pte_val(pte_t pte)
443{
444 pteval_t ret;
445
446 if (sizeof(pteval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800447 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
448 pte.pte, (u64)pte.pte >> 32);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100449 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800450 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
451 pte.pte);
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +0100452
453 return ret;
454}
455
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100456static inline pgd_t __pgd(pgdval_t val)
457{
458 pgdval_t ret;
459
460 if (sizeof(pgdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800461 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
462 val, (u64)val >> 32);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100463 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800464 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
465 val);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100466
467 return (pgd_t) { ret };
468}
469
470static inline pgdval_t pgd_val(pgd_t pgd)
471{
472 pgdval_t ret;
473
474 if (sizeof(pgdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800475 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
476 pgd.pgd, (u64)pgd.pgd >> 32);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100477 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800478 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
479 pgd.pgd);
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +0100480
481 return ret;
482}
483
Jeremy Fitzhardinge08b882c2008-06-16 04:30:01 -0700484#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
485static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
486 pte_t *ptep)
487{
488 pteval_t ret;
489
490 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
491 mm, addr, ptep);
492
493 return (pte_t) { .pte = ret };
494}
495
496static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
497 pte_t *ptep, pte_t pte)
498{
499 if (sizeof(pteval_t) > sizeof(long))
500 /* 5 arg words */
501 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
502 else
503 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
504 mm, addr, ptep, pte.pte);
505}
506
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100507static inline void set_pte(pte_t *ptep, pte_t pte)
508{
509 if (sizeof(pteval_t) > sizeof(long))
510 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
511 pte.pte, (u64)pte.pte >> 32);
512 else
513 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
514 pte.pte);
515}
516
517static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
518 pte_t *ptep, pte_t pte)
519{
520 if (sizeof(pteval_t) > sizeof(long))
521 /* 5 arg words */
522 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
523 else
524 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
525}
526
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800527static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
528 pmd_t *pmdp, pmd_t pmd)
529{
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800530 if (sizeof(pmdval_t) > sizeof(long))
531 /* 5 arg words */
532 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd);
533 else
Andrea Arcangelicacf0612011-01-25 15:07:09 -0800534 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp,
535 native_pmd_val(pmd));
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800536}
Andrea Arcangeli331127f2011-01-13 15:46:36 -0800537
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +0100538static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
539{
540 pmdval_t val = native_pmd_val(pmd);
541
542 if (sizeof(pmdval_t) > sizeof(long))
543 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
544 else
545 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
546}
547
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700548#if CONFIG_PGTABLE_LEVELS >= 3
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100549static inline pmd_t __pmd(pmdval_t val)
550{
551 pmdval_t ret;
552
553 if (sizeof(pmdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800554 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
555 val, (u64)val >> 32);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100556 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800557 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
558 val);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100559
560 return (pmd_t) { ret };
561}
562
563static inline pmdval_t pmd_val(pmd_t pmd)
564{
565 pmdval_t ret;
566
567 if (sizeof(pmdval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800568 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
569 pmd.pmd, (u64)pmd.pmd >> 32);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100570 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800571 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
572 pmd.pmd);
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100573
574 return ret;
575}
576
577static inline void set_pud(pud_t *pudp, pud_t pud)
578{
579 pudval_t val = native_pud_val(pud);
580
581 if (sizeof(pudval_t) > sizeof(long))
582 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
583 val, (u64)val >> 32);
584 else
585 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
586 val);
587}
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700588#if CONFIG_PGTABLE_LEVELS == 4
Eduardo Habkost90422192008-01-30 13:33:20 +0100589static inline pud_t __pud(pudval_t val)
590{
591 pudval_t ret;
592
593 if (sizeof(pudval_t) > sizeof(long))
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800594 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
595 val, (u64)val >> 32);
Eduardo Habkost90422192008-01-30 13:33:20 +0100596 else
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800597 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
598 val);
Eduardo Habkost90422192008-01-30 13:33:20 +0100599
600 return (pud_t) { ret };
601}
602
603static inline pudval_t pud_val(pud_t pud)
604{
605 pudval_t ret;
606
607 if (sizeof(pudval_t) > sizeof(long))
Jeremy Fitzhardinge4767afb2009-01-29 01:51:34 -0800608 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
609 pud.pud, (u64)pud.pud >> 32);
Eduardo Habkost90422192008-01-30 13:33:20 +0100610 else
Jeremy Fitzhardinge4767afb2009-01-29 01:51:34 -0800611 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
612 pud.pud);
Eduardo Habkost90422192008-01-30 13:33:20 +0100613
614 return ret;
615}
616
617static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
618{
619 pgdval_t val = native_pgd_val(pgd);
620
621 if (sizeof(pgdval_t) > sizeof(long))
622 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
623 val, (u64)val >> 32);
624 else
625 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
626 val);
627}
628
629static inline void pgd_clear(pgd_t *pgdp)
630{
631 set_pgd(pgdp, __pgd(0));
632}
633
634static inline void pud_clear(pud_t *pudp)
635{
636 set_pud(pudp, __pud(0));
637}
638
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700639#endif /* CONFIG_PGTABLE_LEVELS == 4 */
Eduardo Habkost90422192008-01-30 13:33:20 +0100640
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700641#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +0100642
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100643#ifdef CONFIG_X86_PAE
644/* Special-case pte-setting operations for PAE, which can't update a
645 64-bit pte atomically */
646static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
647{
648 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
649 pte.pte, pte.pte >> 32);
650}
651
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100652static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
653 pte_t *ptep)
654{
655 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
656}
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +0100657
658static inline void pmd_clear(pmd_t *pmdp)
659{
660 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
661}
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100662#else /* !CONFIG_X86_PAE */
663static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
664{
665 set_pte(ptep, pte);
666}
667
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100668static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
669 pte_t *ptep)
670{
671 set_pte_at(mm, addr, ptep, __pte(0));
672}
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +0100673
674static inline void pmd_clear(pmd_t *pmdp)
675{
676 set_pmd(pmdp, __pmd(0));
677}
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +0100678#endif /* CONFIG_X86_PAE */
679
Jeremy Fitzhardinge7fd7d832009-02-17 23:24:03 -0800680#define __HAVE_ARCH_START_CONTEXT_SWITCH
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800681static inline void arch_start_context_switch(struct task_struct *prev)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200682{
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800683 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200684}
685
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800686static inline void arch_end_context_switch(struct task_struct *next)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200687{
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800688 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200689}
690
Zachary Amsden9226d122007-02-13 13:26:21 +0100691#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200692static inline void arch_enter_lazy_mmu_mode(void)
693{
Jeremy Fitzhardinge8965c1c02007-10-16 11:51:29 -0700694 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200695}
696
697static inline void arch_leave_lazy_mmu_mode(void)
698{
Jeremy Fitzhardinge8965c1c02007-10-16 11:51:29 -0700699 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200700}
701
Boris Ostrovsky511ba862013-03-23 09:36:36 -0400702static inline void arch_flush_lazy_mmu_mode(void)
703{
704 PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush);
705}
Zachary Amsden9226d122007-02-13 13:26:21 +0100706
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700707static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
Masami Hiramatsu3b3809a2009-04-09 10:55:33 -0700708 phys_addr_t phys, pgprot_t flags)
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700709{
710 pv_mmu_ops.set_fixmap(idx, phys, flags);
711}
712
Jeremy Fitzhardingeb4ecc122009-05-13 17:16:55 -0700713#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
Ingo Molnar4bb689e2008-07-09 14:33:33 +0200714
Jeremy Fitzhardinge545ac132013-08-09 19:51:49 +0530715static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
716 __ticket_t ticket)
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700717{
Jeremy Fitzhardinge354714d2013-08-09 19:51:55 +0530718 PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket);
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700719}
720
Jeremy Fitzhardinge96f853e2013-08-09 19:51:58 +0530721static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
Jeremy Fitzhardinge545ac132013-08-09 19:51:49 +0530722 __ticket_t ticket)
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700723{
Jeremy Fitzhardinge545ac132013-08-09 19:51:49 +0530724 PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700725}
726
Ingo Molnar4bb689e2008-07-09 14:33:33 +0200727#endif
728
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100729#ifdef CONFIG_X86_32
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800730#define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
731#define PV_RESTORE_REGS "popl %edx; popl %ecx;"
732
733/* save and restore all caller-save registers, except return value */
Jeremy Fitzhardingee584f552009-01-30 23:17:23 -0800734#define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
735#define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800736
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100737#define PV_FLAGS_ARG "0"
738#define PV_EXTRA_CLOBBERS
739#define PV_VEXTRA_CLOBBERS
740#else
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800741/* save and restore all caller-save registers, except return value */
742#define PV_SAVE_ALL_CALLER_REGS \
743 "push %rcx;" \
744 "push %rdx;" \
745 "push %rsi;" \
746 "push %rdi;" \
747 "push %r8;" \
748 "push %r9;" \
749 "push %r10;" \
750 "push %r11;"
751#define PV_RESTORE_ALL_CALLER_REGS \
752 "pop %r11;" \
753 "pop %r10;" \
754 "pop %r9;" \
755 "pop %r8;" \
756 "pop %rdi;" \
757 "pop %rsi;" \
758 "pop %rdx;" \
759 "pop %rcx;"
760
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100761/* We save some registers, but all of them, that's too much. We clobber all
762 * caller saved registers but the argument parameter */
763#define PV_SAVE_REGS "pushq %%rdi;"
764#define PV_RESTORE_REGS "popq %%rdi;"
Jeremy Fitzhardingec24481e2008-07-08 15:07:12 -0700765#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
766#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100767#define PV_FLAGS_ARG "D"
768#endif
769
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800770/*
771 * Generate a thunk around a function which saves all caller-save
772 * registers except for the return value. This allows C functions to
773 * be called from assembler code where fewer than normal registers are
774 * available. It may also help code generation around calls from C
775 * code if the common case doesn't use many registers.
776 *
777 * When a callee is wrapped in a thunk, the caller can assume that all
778 * arg regs and all scratch registers are preserved across the
779 * call. The return value in rax/eax will not be saved, even for void
780 * functions.
781 */
782#define PV_CALLEE_SAVE_REGS_THUNK(func) \
783 extern typeof(func) __raw_callee_save_##func; \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800784 \
785 asm(".pushsection .text;" \
Andi Kleena2e7f0e2013-10-22 09:07:56 -0700786 ".globl __raw_callee_save_" #func " ; " \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800787 "__raw_callee_save_" #func ": " \
788 PV_SAVE_ALL_CALLER_REGS \
789 "call " #func ";" \
790 PV_RESTORE_ALL_CALLER_REGS \
791 "ret;" \
792 ".popsection")
793
794/* Get a reference to a callee-save function */
795#define PV_CALLEE_SAVE(func) \
796 ((struct paravirt_callee_save) { __raw_callee_save_##func })
797
798/* Promise that "func" already uses the right calling convention */
799#define __PV_IS_CALLEE_SAVE(func) \
800 ((struct paravirt_callee_save) { func })
801
Steven Rostedtb5908542010-11-10 22:29:49 -0500802static inline notrace unsigned long arch_local_save_flags(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100803{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700804 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100805}
806
Steven Rostedtb5908542010-11-10 22:29:49 -0500807static inline notrace void arch_local_irq_restore(unsigned long f)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100808{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700809 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100810}
811
Steven Rostedtb5908542010-11-10 22:29:49 -0500812static inline notrace void arch_local_irq_disable(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100813{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700814 PVOP_VCALLEE0(pv_irq_ops.irq_disable);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100815}
816
Steven Rostedtb5908542010-11-10 22:29:49 -0500817static inline notrace void arch_local_irq_enable(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100818{
Jeremy Fitzhardinge71999d92009-10-12 16:32:43 -0700819 PVOP_VCALLEE0(pv_irq_ops.irq_enable);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100820}
821
Steven Rostedtb5908542010-11-10 22:29:49 -0500822static inline notrace unsigned long arch_local_irq_save(void)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100823{
824 unsigned long f;
825
David Howellsdf9ee292010-10-07 14:08:55 +0100826 f = arch_local_save_flags();
827 arch_local_irq_disable();
Rusty Russell139ec7c2006-12-07 02:14:08 +0100828 return f;
829}
830
Jeremy Fitzhardinge74d4aff2008-07-07 12:07:50 -0700831
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200832/* Make sure as little as possible of this mess escapes. */
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200833#undef PARAVIRT_CALL
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200834#undef __PVOP_CALL
835#undef __PVOP_VCALL
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200836#undef PVOP_VCALL0
837#undef PVOP_CALL0
838#undef PVOP_VCALL1
839#undef PVOP_CALL1
840#undef PVOP_VCALL2
841#undef PVOP_CALL2
842#undef PVOP_VCALL3
843#undef PVOP_CALL3
844#undef PVOP_VCALL4
845#undef PVOP_CALL4
Rusty Russell139ec7c2006-12-07 02:14:08 +0100846
Thomas Gleixner6f30c1a2009-08-20 13:19:57 +0200847extern void default_banner(void);
848
Rusty Russelld3561b72006-12-07 02:14:07 +0100849#else /* __ASSEMBLY__ */
850
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100851#define _PVSITE(ptype, clobbers, ops, word, algn) \
Rusty Russell139ec7c2006-12-07 02:14:08 +0100852771:; \
853 ops; \
854772:; \
855 .pushsection .parainstructions,"a"; \
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100856 .align algn; \
857 word 771b; \
Rusty Russell139ec7c2006-12-07 02:14:08 +0100858 .byte ptype; \
859 .byte 772b-771b; \
860 .short clobbers; \
861 .popsection
862
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100863
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800864#define COND_PUSH(set, mask, reg) \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800865 .if ((~(set)) & mask); push %reg; .endif
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800866#define COND_POP(set, mask, reg) \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800867 .if ((~(set)) & mask); pop %reg; .endif
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800868
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100869#ifdef CONFIG_X86_64
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800870
871#define PV_SAVE_REGS(set) \
872 COND_PUSH(set, CLBR_RAX, rax); \
873 COND_PUSH(set, CLBR_RCX, rcx); \
874 COND_PUSH(set, CLBR_RDX, rdx); \
875 COND_PUSH(set, CLBR_RSI, rsi); \
876 COND_PUSH(set, CLBR_RDI, rdi); \
877 COND_PUSH(set, CLBR_R8, r8); \
878 COND_PUSH(set, CLBR_R9, r9); \
879 COND_PUSH(set, CLBR_R10, r10); \
880 COND_PUSH(set, CLBR_R11, r11)
881#define PV_RESTORE_REGS(set) \
882 COND_POP(set, CLBR_R11, r11); \
883 COND_POP(set, CLBR_R10, r10); \
884 COND_POP(set, CLBR_R9, r9); \
885 COND_POP(set, CLBR_R8, r8); \
886 COND_POP(set, CLBR_RDI, rdi); \
887 COND_POP(set, CLBR_RSI, rsi); \
888 COND_POP(set, CLBR_RDX, rdx); \
889 COND_POP(set, CLBR_RCX, rcx); \
890 COND_POP(set, CLBR_RAX, rax)
891
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +0100892#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100893#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400894#define PARA_INDIRECT(addr) *addr(%rip)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100895#else
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800896#define PV_SAVE_REGS(set) \
897 COND_PUSH(set, CLBR_EAX, eax); \
898 COND_PUSH(set, CLBR_EDI, edi); \
899 COND_PUSH(set, CLBR_ECX, ecx); \
900 COND_PUSH(set, CLBR_EDX, edx)
901#define PV_RESTORE_REGS(set) \
902 COND_POP(set, CLBR_EDX, edx); \
903 COND_POP(set, CLBR_ECX, ecx); \
904 COND_POP(set, CLBR_EDI, edi); \
905 COND_POP(set, CLBR_EAX, eax)
906
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +0100907#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100908#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400909#define PARA_INDIRECT(addr) *%cs:addr
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100910#endif
911
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700912#define INTERRUPT_RETURN \
913 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400914 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
Rusty Russell139ec7c2006-12-07 02:14:08 +0100915
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200916#define DISABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700917 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800918 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400919 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800920 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100921
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200922#define ENABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700923 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800924 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400925 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800926 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100927
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400928#define USERGS_SYSRET32 \
929 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
Glauber de Oliveira Costa6abcd982008-01-30 13:30:33 +0100930 CLBR_NONE, \
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400931 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +0100932
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +0100933#ifdef CONFIG_X86_32
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -0400934#define GET_CR0_INTO_EAX \
935 push %ecx; push %edx; \
936 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
Jeremy Fitzhardinge42c24fa2007-05-02 19:27:14 +0200937 pop %edx; pop %ecx
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400938
939#define ENABLE_INTERRUPTS_SYSEXIT \
940 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
941 CLBR_NONE, \
942 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
943
944
945#else /* !CONFIG_X86_32 */
Jeremy Fitzhardingea00394f2008-06-25 00:19:30 -0400946
947/*
948 * If swapgs is used while the userspace stack is still current,
949 * there's no way to call a pvop. The PV replacement *must* be
950 * inlined, or the swapgs instruction must be trapped and emulated.
951 */
952#define SWAPGS_UNSAFE_STACK \
953 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
954 swapgs)
955
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800956/*
957 * Note: swapgs is very special, and in practise is either going to be
958 * implemented with a single "swapgs" instruction or something very
959 * special. Either way, we don't need to save any registers for
960 * it.
961 */
Glauber de Oliveira Costae801f862008-01-30 13:32:08 +0100962#define SWAPGS \
963 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
Jeremy Fitzhardinge9104a182009-01-28 14:35:04 -0800964 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \
Glauber de Oliveira Costae801f862008-01-30 13:32:08 +0100965 )
966
H. Peter Anvinffc4bc92012-04-18 17:16:48 -0700967#define GET_CR2_INTO_RAX \
968 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2)
Glauber de Oliveira Costa4a8c4c42008-01-30 13:32:07 +0100969
Jeremy Fitzhardingefab58422008-06-25 00:19:31 -0400970#define PARAVIRT_ADJUST_EXCEPTION_FRAME \
971 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
972 CLBR_NONE, \
973 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
974
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400975#define USERGS_SYSRET64 \
976 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \
Jeremy Fitzhardinged75cd222008-06-25 00:19:26 -0400977 CLBR_NONE, \
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400978 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400979#endif /* CONFIG_X86_32 */
Rusty Russell139ec7c2006-12-07 02:14:08 +0100980
Rusty Russelld3561b72006-12-07 02:14:07 +0100981#endif /* __ASSEMBLY__ */
Thomas Gleixner6f30c1a2009-08-20 13:19:57 +0200982#else /* CONFIG_PARAVIRT */
983# define default_banner x86_init_noop
Dave Hansena1ea1c02014-11-18 10:23:49 -0800984#ifndef __ASSEMBLY__
985static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
986 struct mm_struct *mm)
987{
988}
989
990static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
991{
992}
993#endif /* __ASSEMBLY__ */
Thomas Gleixner6f30c1a2009-08-20 13:19:57 +0200994#endif /* !CONFIG_PARAVIRT */
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700995#endif /* _ASM_X86_PARAVIRT_H */