blob: bc678f0c3c9174a2d3d3599bd47d9a8aca42e453 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Ingo Molnar54ad7262015-06-05 13:02:28 +02002 * Compatibility mode system call entry point for x86-64.
3 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
Ingo Molnar54ad7262015-06-05 13:02:28 +02005 */
Ingo Molnard36f9472015-06-03 18:29:26 +02006#include "calling.h"
Sam Ravnborge2d5df92005-09-09 21:28:48 +02007#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <asm/current.h>
9#include <asm/errno.h>
Ingo Molnar54ad7262015-06-05 13:02:28 +020010#include <asm/ia32_unistd.h>
11#include <asm/thread_info.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <asm/segment.h>
Ingo Molnar2601e642006-07-03 00:24:45 -070013#include <asm/irqflags.h>
H. Peter Anvin1ce6f862012-04-20 12:19:50 -070014#include <asm/asm.h>
H. Peter Anvin63bcff22012-09-21 12:43:12 -070015#include <asm/smap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/linkage.h>
Eric Parisd7e75282012-01-03 14:23:06 -050017#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
Roland McGrath5cbf1562008-06-24 01:13:31 -070019/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
20#include <linux/elf-em.h>
21#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
Ingo Molnar54ad7262015-06-05 13:02:28 +020022#define __AUDIT_ARCH_LE 0x40000000
Roland McGrath5cbf1562008-06-24 01:13:31 -070023
24#ifndef CONFIG_AUDITSYSCALL
Andy Lutomirski8f7f06b2015-07-07 10:55:28 -070025# define sysexit_audit ia32_ret_from_sys_call_irqs_off
26# define sysretl_audit ia32_ret_from_sys_call_irqs_off
Roland McGrath5cbf1562008-06-24 01:13:31 -070027#endif
28
Jiri Olsaea714542011-03-07 19:10:39 +010029 .section .entry.text, "ax"
30
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -040031#ifdef CONFIG_PARAVIRT
32ENTRY(native_usergs_sysret32)
33 swapgs
34 sysretl
35ENDPROC(native_usergs_sysret32)
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -040036#endif
37
Linus Torvalds1da177e2005-04-16 15:20:36 -070038/*
Ingo Molnar54ad7262015-06-05 13:02:28 +020039 * 32-bit SYSENTER instruction entry.
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 *
Denys Vlasenkob87cf632015-02-26 14:40:32 -080041 * SYSENTER loads ss, rsp, cs, and rip from previously programmed MSRs.
42 * IF and VM in rflags are cleared (IOW: interrupts are off).
43 * SYSENTER does not save anything on the stack,
44 * and does not save old rip (!!!) and rflags.
45 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * Arguments:
Denys Vlasenkob87cf632015-02-26 14:40:32 -080047 * eax system call number
48 * ebx arg1
49 * ecx arg2
50 * edx arg3
51 * esi arg4
52 * edi arg5
53 * ebp user stack
54 * 0(%ebp) arg6
55 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 * This is purely a fast path. For anything complicated we use the int 0x80
Denys Vlasenkob87cf632015-02-26 14:40:32 -080057 * path below. We set up a complete hardware stack frame to share code
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 * with the int 0x80 path.
Denys Vlasenkob87cf632015-02-26 14:40:32 -080059 */
Ingo Molnar4c8cd0c2015-06-08 08:33:56 +020060ENTRY(entry_SYSENTER_compat)
Andy Lutomirskib611acf2015-10-05 17:47:55 -070061 /* Interrupts are off on entry. */
Jeremy Fitzhardinge457da702008-06-26 07:28:51 -070062 SWAPGS_UNSAFE_STACK
Denys Vlasenko3a232082015-04-24 17:31:35 +020063 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +010064
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +010065 /* Zero-extending 32-bit regs, do not remove */
66 movl %ebp, %ebp
67 movl %eax, %eax
68
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020069 movl ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020070
71 /* Construct struct pt_regs on stack */
Ingo Molnar131484c2015-05-28 12:21:47 +020072 pushq $__USER32_DS /* pt_regs->ss */
73 pushq %rbp /* pt_regs->sp */
Andy Lutomirskib611acf2015-10-05 17:47:55 -070074
75 /*
76 * Push flags. This is nasty. First, interrupts are currently
77 * off, but we need pt_regs->flags to have IF set. Second, even
78 * if TF was set when SYSENTER started, it's clear by now. We fix
79 * that later using TIF_SINGLESTEP.
80 */
81 pushfq /* pt_regs->flags (except IF = 0) */
82 orl $X86_EFLAGS_IF, (%rsp) /* Fix saved flags */
83
Ingo Molnar131484c2015-05-28 12:21:47 +020084 pushq $__USER32_CS /* pt_regs->cs */
Ingo Molnar54ad7262015-06-05 13:02:28 +020085 pushq %r10 /* pt_regs->ip = thread_info->sysenter_return */
Ingo Molnar131484c2015-05-28 12:21:47 +020086 pushq %rax /* pt_regs->orig_ax */
87 pushq %rdi /* pt_regs->di */
88 pushq %rsi /* pt_regs->si */
89 pushq %rdx /* pt_regs->dx */
90 pushq %rcx /* pt_regs->cx */
91 pushq $-ENOSYS /* pt_regs->ax */
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 cld
Ingo Molnar54ad7262015-06-05 13:02:28 +020093 sub $(10*8), %rsp /* pt_regs->r8-11, bp, bx, r12-15 not saved */
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020094
Denys Vlasenkob87cf632015-02-26 14:40:32 -080095 /*
Andy Lutomirski8c7aa692014-10-01 11:49:04 -070096 * Sysenter doesn't filter flags, so we need to clear NT
97 * ourselves. To save a few cycles, we can check whether
98 * NT was set instead of doing an unconditional popfq.
Andy Lutomirskib611acf2015-10-05 17:47:55 -070099 * This needs to happen before enabling interrupts so that
100 * we don't get preempted with NT set.
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700101 */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200102 testl $X86_EFLAGS_NT, EFLAGS(%rsp)
103 jnz sysenter_fix_flags
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700104sysenter_flags_fixed:
105
Andy Lutomirskidd27f992015-10-05 17:47:53 -0700106 /*
Andy Lutomirskib611acf2015-10-05 17:47:55 -0700107 * Re-enable interrupts. IRQ tracing already thinks that IRQs are
108 * on (since we treat user mode as having IRQs on), and the
109 * prologue above is too short for it to be worth adding a
110 * tracing round trip.
111 */
112 ENABLE_INTERRUPTS(CLBR_NONE)
113
114 /*
Andy Lutomirskidd27f992015-10-05 17:47:53 -0700115 * No need to do an access_ok() check here because RBP has been
116 * 32-bit zero extended:
117 */
118 ASM_STAC
1191: movl (%rbp), %ebp
120 _ASM_EXTABLE(1b, ia32_badarg)
121 ASM_CLAC
122
Denys Vlasenko9b47feb2015-06-08 22:35:33 +0200123 orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
124 testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200125 jnz sysenter_tracesys
Ingo Molnar131484c2015-05-28 12:21:47 +0200126
Roland McGrathd4d67152008-07-09 02:38:07 -0700127sysenter_do_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200128 /* 32-bit syscall -> 64-bit C ABI argument conversion */
129 movl %edi, %r8d /* arg5 */
130 movl %ebp, %r9d /* arg6 */
131 xchg %ecx, %esi /* rsi:arg2, rcx:arg4 */
132 movl %ebx, %edi /* arg1 */
133 movl %edx, %edx /* arg3 (zero extension) */
Roland McGrath5cbf1562008-06-24 01:13:31 -0700134sysenter_dispatch:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200135 cmpq $(IA32_NR_syscalls-1), %rax
Denys Vlasenko3f5159a2015-04-21 18:03:14 +0200136 ja 1f
Ingo Molnar54ad7262015-06-05 13:02:28 +0200137 call *ia32_sys_call_table(, %rax, 8)
138 movq %rax, RAX(%rsp)
Denys Vlasenko3f5159a2015-04-21 18:03:14 +02001391:
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400140 DISABLE_INTERRUPTS(CLBR_NONE)
Ingo Molnar2601e642006-07-03 00:24:45 -0700141 TRACE_IRQS_OFF
Ingo Molnardca5b522015-03-24 19:44:42 +0100142 testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
Roland McGrath5cbf1562008-06-24 01:13:31 -0700143 jnz sysexit_audit
144sysexit_from_sys_call:
Andy Lutomirski4214a162015-04-02 17:12:12 -0700145 /*
146 * NB: SYSEXIT is not obviously safe for 64-bit kernels -- an
147 * NMI between STI and SYSEXIT has poorly specified behavior,
148 * and and NMI followed by an IRQ with usergs is fatal. So
149 * we just pretend we're using SYSEXIT but we really use
150 * SYSRETL instead.
151 *
152 * This code path is still called 'sysexit' because it pairs
153 * with 'sysenter' and it uses the SYSENTER calling convention.
154 */
Denys Vlasenko9b47feb2015-06-08 22:35:33 +0200155 andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200156 movl RIP(%rsp), %ecx /* User %eip */
Linus Torvaldscd88ec22015-08-13 16:19:44 -0700157 movq RAX(%rsp), %rax
Denys Vlasenkoc73e36b2015-07-03 22:19:02 +0200158 movl RSI(%rsp), %esi
159 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200160 xorl %edx, %edx /* Do not leak kernel information */
161 xorq %r8, %r8
162 xorq %r9, %r9
163 xorq %r10, %r10
164 movl EFLAGS(%rsp), %r11d /* User eflags */
Ingo Molnar2601e642006-07-03 00:24:45 -0700165 TRACE_IRQS_ON
Andy Lutomirski4214a162015-04-02 17:12:12 -0700166
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800167 /*
Andy Lutomirski4214a162015-04-02 17:12:12 -0700168 * SYSRETL works even on Intel CPUs. Use it in preference to SYSEXIT,
169 * since it avoids a dicey window with interrupts enabled.
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800170 */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200171 movl RSP(%rsp), %esp
Andy Lutomirski4214a162015-04-02 17:12:12 -0700172
173 /*
174 * USERGS_SYSRET32 does:
175 * gsbase = user's gs base
176 * eip = ecx
177 * rflags = r11
178 * cs = __USER32_CS
179 * ss = __USER_DS
180 *
181 * The prologue set RIP(%rsp) to VDSO32_SYSENTER_RETURN, which does:
182 *
183 * pop %ebp
184 * pop %edx
185 * pop %ecx
186 *
187 * Therefore, we invoke SYSRETL with EDX and R8-R10 zeroed to
188 * avoid info leaks. R11 ends up with VDSO32_SYSENTER_RETURN's
189 * address (already known to user code), and R12-R15 are
190 * callee-saved and therefore don't contain any interesting
191 * kernel data.
192 */
193 USERGS_SYSRET32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194
Roland McGrath5cbf1562008-06-24 01:13:31 -0700195#ifdef CONFIG_AUDITSYSCALL
196 .macro auditsys_entry_common
Denys Vlasenko1536bb42015-06-09 20:54:08 +0200197 /*
Denys Vlasenkoa92fde252015-06-09 20:54:09 +0200198 * At this point, registers hold syscall args in the 32-bit syscall ABI:
199 * EAX is syscall number, the 6 args are in EBX,ECX,EDX,ESI,EDI,EBP.
200 *
201 * We want to pass them to __audit_syscall_entry(), which is a 64-bit
202 * C function with 5 parameters, so shuffle them to match what
203 * the function expects: RDI,RSI,RDX,RCX,R8.
204 */
205 movl %esi, %r8d /* arg5 (R8 ) <= 4th syscall arg (ESI) */
206 xchg %ecx, %edx /* arg4 (RCX) <= 3rd syscall arg (EDX) */
207 /* arg3 (RDX) <= 2nd syscall arg (ECX) */
208 movl %ebx, %esi /* arg2 (RSI) <= 1st syscall arg (EBX) */
209 movl %eax, %edi /* arg1 (RDI) <= syscall number (EAX) */
210 call __audit_syscall_entry
211
212 /*
213 * We are going to jump back to the syscall dispatch code.
214 * Prepare syscall args as required by the 64-bit C ABI.
215 * Registers clobbered by __audit_syscall_entry() are
216 * loaded from pt_regs on stack:
Denys Vlasenko1536bb42015-06-09 20:54:08 +0200217 */
218 movl ORIG_RAX(%rsp), %eax /* syscall number */
219 movl %ebx, %edi /* arg1 */
220 movl RCX(%rsp), %esi /* arg2 */
221 movl RDX(%rsp), %edx /* arg3 */
222 movl RSI(%rsp), %ecx /* arg4 */
223 movl RDI(%rsp), %r8d /* arg5 */
Roland McGrath5cbf1562008-06-24 01:13:31 -0700224 .endm
225
Jan Beulich81766742009-10-26 15:20:29 +0000226 .macro auditsys_exit exit
Roland McGrath5cbf1562008-06-24 01:13:31 -0700227 TRACE_IRQS_ON
Jan Beulich40a1ef92013-01-30 07:55:53 +0000228 ENABLE_INTERRUPTS(CLBR_NONE)
Andy Lutomirski29ea1b22015-07-03 12:44:28 -0700229 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
230 jnz ia32_ret_from_sys_call
Ingo Molnar54ad7262015-06-05 13:02:28 +0200231 movl %eax, %esi /* second arg, syscall return value */
232 cmpl $-MAX_ERRNO, %eax /* is it an error ? */
233 jbe 1f
234 movslq %eax, %rsi /* if error sign extend to 64 bits */
2351: setbe %al /* 1 if error, 0 if not */
236 movzbl %al, %edi /* zero-extend that into %edi */
237 call __audit_syscall_exit
Ingo Molnar54ad7262015-06-05 13:02:28 +0200238 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %edi
Jan Beulich40a1ef92013-01-30 07:55:53 +0000239 DISABLE_INTERRUPTS(CLBR_NONE)
Roland McGrath5cbf1562008-06-24 01:13:31 -0700240 TRACE_IRQS_OFF
Ingo Molnar54ad7262015-06-05 13:02:28 +0200241 testl %edi, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
242 jz \exit
243 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200244 movq %rax, R11(%rsp)
245 movq %rax, R10(%rsp)
246 movq %rax, R9(%rsp)
247 movq %rax, R8(%rsp)
Andy Lutomirski29ea1b22015-07-03 12:44:28 -0700248 jmp int_ret_from_sys_call_irqs_off
Roland McGrath5cbf1562008-06-24 01:13:31 -0700249 .endm
250
251sysenter_auditsys:
Roland McGrath5cbf1562008-06-24 01:13:31 -0700252 auditsys_entry_common
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200253 movl %ebp, %r9d /* reload 6th syscall arg */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200254 jmp sysenter_dispatch
Roland McGrath5cbf1562008-06-24 01:13:31 -0700255
256sysexit_audit:
257 auditsys_exit sysexit_from_sys_call
258#endif
259
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700260sysenter_fix_flags:
Andy Lutomirskib611acf2015-10-05 17:47:55 -0700261 pushq $X86_EFLAGS_FIXED
Ingo Molnar131484c2015-05-28 12:21:47 +0200262 popfq
Ingo Molnar54ad7262015-06-05 13:02:28 +0200263 jmp sysenter_flags_fixed
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700264
Roland McGrath5cbf1562008-06-24 01:13:31 -0700265sysenter_tracesys:
Roland McGrath5cbf1562008-06-24 01:13:31 -0700266#ifdef CONFIG_AUDITSYSCALL
Ingo Molnardca5b522015-03-24 19:44:42 +0100267 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
Roland McGrath5cbf1562008-06-24 01:13:31 -0700268 jz sysenter_auditsys
269#endif
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800270 SAVE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200271 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200272 movq %rax, R11(%rsp)
273 movq %rax, R10(%rsp)
274 movq %rax, R9(%rsp)
275 movq %rax, R8(%rsp)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200276 movq %rsp, %rdi /* &pt_regs -> arg1 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 call syscall_trace_enter
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200278
279 /* Reload arg registers from stack. (see sysenter_tracesys) */
280 movl RCX(%rsp), %ecx
281 movl RDX(%rsp), %edx
282 movl RSI(%rsp), %esi
283 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200284 movl %eax, %eax /* zero extension */
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200285
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800286 RESTORE_EXTRA_REGS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 jmp sysenter_do_call
Ingo Molnar4c8cd0c2015-06-08 08:33:56 +0200288ENDPROC(entry_SYSENTER_compat)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289
290/*
Ingo Molnar54ad7262015-06-05 13:02:28 +0200291 * 32-bit SYSCALL instruction entry.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 *
Ingo Molnar54ad7262015-06-05 13:02:28 +0200293 * 32-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800294 * then loads new ss, cs, and rip from previously programmed MSRs.
295 * rflags gets masked by a value from another MSR (so CLD and CLAC
296 * are not needed). SYSCALL does not save anything on the stack
297 * and does not change rsp.
298 *
299 * Note: rflags saving+masking-with-MSR happens only in Long mode
Ingo Molnar54ad7262015-06-05 13:02:28 +0200300 * (in legacy 32-bit mode, IF, RF and VM bits are cleared and that's it).
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800301 * Don't get confused: rflags saving+masking depends on Long Mode Active bit
302 * (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
303 * or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
304 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 * Arguments:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800306 * eax system call number
307 * ecx return address
308 * ebx arg1
309 * ebp arg2 (note: not saved in the stack frame, should not be touched)
310 * edx arg3
311 * esi arg4
312 * edi arg5
313 * esp user stack
314 * 0(%esp) arg6
315 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 * This is purely a fast path. For anything complicated we use the int 0x80
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800317 * path below. We set up a complete hardware stack frame to share code
318 * with the int 0x80 path.
319 */
Ingo Molnar2cd23552015-06-08 08:28:07 +0200320ENTRY(entry_SYSCALL_compat)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100321 /*
322 * Interrupts are off on entry.
323 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
324 * it is too small to ever cause noticeable irq latency.
325 */
Jeremy Fitzhardinge457da702008-06-26 07:28:51 -0700326 SWAPGS_UNSAFE_STACK
Ingo Molnar54ad7262015-06-05 13:02:28 +0200327 movl %esp, %r8d
328 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400329 ENABLE_INTERRUPTS(CLBR_NONE)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100330
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100331 /* Zero-extending 32-bit regs, do not remove */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200332 movl %eax, %eax
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100333
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200334 /* Construct struct pt_regs on stack */
Ingo Molnar131484c2015-05-28 12:21:47 +0200335 pushq $__USER32_DS /* pt_regs->ss */
336 pushq %r8 /* pt_regs->sp */
337 pushq %r11 /* pt_regs->flags */
338 pushq $__USER32_CS /* pt_regs->cs */
339 pushq %rcx /* pt_regs->ip */
340 pushq %rax /* pt_regs->orig_ax */
341 pushq %rdi /* pt_regs->di */
342 pushq %rsi /* pt_regs->si */
343 pushq %rdx /* pt_regs->dx */
344 pushq %rbp /* pt_regs->cx */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200345 movl %ebp, %ecx
Ingo Molnar131484c2015-05-28 12:21:47 +0200346 pushq $-ENOSYS /* pt_regs->ax */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200347 sub $(10*8), %rsp /* pt_regs->r8-11, bp, bx, r12-15 not saved */
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200348
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800349 /*
Ingo Molnar54ad7262015-06-05 13:02:28 +0200350 * No need to do an access_ok check here because r8 has been
351 * 32-bit zero extended:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800352 */
H. Peter Anvin63bcff22012-09-21 12:43:12 -0700353 ASM_STAC
Denys Vlasenkoc0c33222015-07-24 14:16:43 +02003541: movl (%r8), %r9d
Ingo Molnar54ad7262015-06-05 13:02:28 +0200355 _ASM_EXTABLE(1b, ia32_badarg)
H. Peter Anvin63bcff22012-09-21 12:43:12 -0700356 ASM_CLAC
Denys Vlasenko9b47feb2015-06-08 22:35:33 +0200357 orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
358 testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
359 jnz cstar_tracesys
Ingo Molnar131484c2015-05-28 12:21:47 +0200360
Jan Beulich295286a82008-08-29 13:21:11 +0100361cstar_do_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200362 /* 32-bit syscall -> 64-bit C ABI argument conversion */
363 movl %edi, %r8d /* arg5 */
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200364 /* r9 already loaded */ /* arg6 */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200365 xchg %ecx, %esi /* rsi:arg2, rcx:arg4 */
366 movl %ebx, %edi /* arg1 */
367 movl %edx, %edx /* arg3 (zero extension) */
368
Roland McGrath5cbf1562008-06-24 01:13:31 -0700369cstar_dispatch:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200370 cmpq $(IA32_NR_syscalls-1), %rax
Denys Vlasenko3f5159a2015-04-21 18:03:14 +0200371 ja 1f
Ingo Molnar54ad7262015-06-05 13:02:28 +0200372
373 call *ia32_sys_call_table(, %rax, 8)
374 movq %rax, RAX(%rsp)
Denys Vlasenko3f5159a2015-04-21 18:03:14 +02003751:
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400376 DISABLE_INTERRUPTS(CLBR_NONE)
Ingo Molnar2601e642006-07-03 00:24:45 -0700377 TRACE_IRQS_OFF
Ingo Molnar54ad7262015-06-05 13:02:28 +0200378 testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
379 jnz sysretl_audit
380
Roland McGrath5cbf1562008-06-24 01:13:31 -0700381sysretl_from_sys_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200382 andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
Denys Vlasenkoc73e36b2015-07-03 22:19:02 +0200383 movl RDX(%rsp), %edx
384 movl RSI(%rsp), %esi
385 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200386 movl RIP(%rsp), %ecx
387 movl EFLAGS(%rsp), %r11d
Linus Torvaldscd88ec22015-08-13 16:19:44 -0700388 movq RAX(%rsp), %rax
Ingo Molnar54ad7262015-06-05 13:02:28 +0200389 xorq %r10, %r10
390 xorq %r9, %r9
391 xorq %r8, %r8
Ingo Molnar2601e642006-07-03 00:24:45 -0700392 TRACE_IRQS_ON
Ingo Molnar54ad7262015-06-05 13:02:28 +0200393 movl RSP(%rsp), %esp
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800394 /*
Ingo Molnar54ad7262015-06-05 13:02:28 +0200395 * 64-bit->32-bit SYSRET restores eip from ecx,
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800396 * eflags from r11 (but RF and VM bits are forced to 0),
397 * cs and ss are loaded from MSRs.
Ingo Molnar54ad7262015-06-05 13:02:28 +0200398 * (Note: 32-bit->32-bit SYSRET is different: since r11
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800399 * does not exist, it merely sets eflags.IF=1).
Andy Lutomirski61f01dd2015-04-26 16:47:59 -0700400 *
401 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
402 * descriptor is not reinitialized. This means that we must
403 * avoid SYSRET with SS == NULL, which could happen if we schedule,
404 * exit the kernel, and re-enter using an interrupt vector. (All
405 * interrupt entries on x86_64 set SS to NULL.) We prevent that
406 * from happening by reloading SS in __switch_to.
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800407 */
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400408 USERGS_SYSRET32
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800409
Roland McGrath5cbf1562008-06-24 01:13:31 -0700410#ifdef CONFIG_AUDITSYSCALL
411cstar_auditsys:
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200412 movl %r9d, R9(%rsp) /* register to be clobbered by call */
Roland McGrath5cbf1562008-06-24 01:13:31 -0700413 auditsys_entry_common
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200414 movl R9(%rsp), %r9d /* reload 6th syscall arg */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200415 jmp cstar_dispatch
Roland McGrath5cbf1562008-06-24 01:13:31 -0700416
417sysretl_audit:
Jan Beulich81766742009-10-26 15:20:29 +0000418 auditsys_exit sysretl_from_sys_call
Roland McGrath5cbf1562008-06-24 01:13:31 -0700419#endif
420
421cstar_tracesys:
422#ifdef CONFIG_AUDITSYSCALL
Ingo Molnar54ad7262015-06-05 13:02:28 +0200423 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
424 jz cstar_auditsys
Roland McGrath5cbf1562008-06-24 01:13:31 -0700425#endif
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200426 xchgl %r9d, %ebp
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800427 SAVE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200428 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200429 movq %rax, R11(%rsp)
430 movq %rax, R10(%rsp)
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200431 movq %r9, R9(%rsp)
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200432 movq %rax, R8(%rsp)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200433 movq %rsp, %rdi /* &pt_regs -> arg1 */
Denys Vlasenko53e9acc2015-06-03 14:56:09 +0200434 call syscall_trace_enter
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200435 movl R9(%rsp), %r9d
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200436
437 /* Reload arg registers from stack. (see sysenter_tracesys) */
438 movl RCX(%rsp), %ecx
439 movl RDX(%rsp), %edx
440 movl RSI(%rsp), %esi
441 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200442 movl %eax, %eax /* zero extension */
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200443
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800444 RESTORE_EXTRA_REGS
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200445 xchgl %ebp, %r9d
Denys Vlasenko53e9acc2015-06-03 14:56:09 +0200446 jmp cstar_do_call
Ingo Molnar2cd23552015-06-08 08:28:07 +0200447END(entry_SYSCALL_compat)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200448
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449ia32_badarg:
Andy Lutomirski5e99cb72015-07-03 12:44:19 -0700450 /*
451 * So far, we've entered kernel mode, set AC, turned on IRQs, and
452 * saved C regs except r8-r11. We haven't done any of the other
453 * standard entry work, though. We want to bail, but we shouldn't
454 * treat this as a syscall entry since we don't even know what the
455 * args are. Instead, treat this as a non-syscall entry, finish
456 * the entry work, and immediately exit after setting AX = -EFAULT.
457 *
458 * We're really just being polite here. Killing the task outright
459 * would be a reasonable action, too. Given that the only valid
460 * way to have gotten here is through the vDSO, and we already know
461 * that the stack pointer is bad, the task isn't going to survive
462 * for long no matter what we do.
463 */
464
465 ASM_CLAC /* undo STAC */
466 movq $-EFAULT, RAX(%rsp) /* return -EFAULT if possible */
467
468 /* Fill in the rest of pt_regs */
469 xorl %eax, %eax
470 movq %rax, R11(%rsp)
471 movq %rax, R10(%rsp)
472 movq %rax, R9(%rsp)
473 movq %rax, R8(%rsp)
474 SAVE_EXTRA_REGS
475
476 /* Turn IRQs back off. */
477 DISABLE_INTERRUPTS(CLBR_NONE)
478 TRACE_IRQS_OFF
479
Andy Lutomirski02bc7762015-07-03 12:44:31 -0700480 /* Now finish entering normal kernel mode. */
481#ifdef CONFIG_CONTEXT_TRACKING
482 call enter_from_user_mode
483#endif
484
Andy Lutomirski5e99cb72015-07-03 12:44:19 -0700485 /* And exit again. */
486 jmp retint_user
487
Andy Lutomirski8f7f06b2015-07-07 10:55:28 -0700488ia32_ret_from_sys_call_irqs_off:
489 TRACE_IRQS_ON
490 ENABLE_INTERRUPTS(CLBR_NONE)
491
Denys Vlasenko61b1e3e2015-06-02 19:35:10 +0200492ia32_ret_from_sys_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200493 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200494 movq %rax, R11(%rsp)
495 movq %rax, R10(%rsp)
496 movq %rax, R9(%rsp)
497 movq %rax, R8(%rsp)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200498 jmp int_ret_from_sys_call
Denys Vlasenko61b1e3e2015-06-02 19:35:10 +0200499
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800500/*
501 * Emulated IA32 system calls via int 0x80.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 *
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800503 * Arguments:
504 * eax system call number
505 * ebx arg1
506 * ecx arg2
507 * edx arg3
508 * esi arg4
509 * edi arg5
510 * ebp arg6 (note: not saved in the stack frame, should not be touched)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 *
512 * Notes:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800513 * Uses the same stack frame as the x86-64 version.
514 * All registers except eax must be saved (but ptrace may violate that).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 * Arguments are zero extended. For system calls that want sign extension and
516 * take long arguments a wrapper is needed. Most calls can just be called
517 * directly.
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800518 * Assumes it is only called from user space and entered with interrupts off.
519 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520
Ingo Molnar2cd23552015-06-08 08:28:07 +0200521ENTRY(entry_INT80_compat)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100522 /*
523 * Interrupts are off on entry.
524 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
525 * it is too small to ever cause noticeable irq latency.
526 */
Jeremy Fitzhardinge360c0442008-07-08 15:06:28 -0700527 PARAVIRT_ADJUST_EXCEPTION_FRAME
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400528 SWAPGS
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400529 ENABLE_INTERRUPTS(CLBR_NONE)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100530
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100531 /* Zero-extending 32-bit regs, do not remove */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200532 movl %eax, %eax
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100533
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200534 /* Construct struct pt_regs on stack (iret frame is already on stack) */
Ingo Molnar131484c2015-05-28 12:21:47 +0200535 pushq %rax /* pt_regs->orig_ax */
536 pushq %rdi /* pt_regs->di */
537 pushq %rsi /* pt_regs->si */
538 pushq %rdx /* pt_regs->dx */
539 pushq %rcx /* pt_regs->cx */
540 pushq $-ENOSYS /* pt_regs->ax */
Denys Vlasenko61b1e3e2015-06-02 19:35:10 +0200541 pushq $0 /* pt_regs->r8 */
542 pushq $0 /* pt_regs->r9 */
543 pushq $0 /* pt_regs->r10 */
544 pushq $0 /* pt_regs->r11 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 cld
Ingo Molnar54ad7262015-06-05 13:02:28 +0200546 sub $(6*8), %rsp /* pt_regs->bp, bx, r12-15 not saved */
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200547
Ingo Molnar54ad7262015-06-05 13:02:28 +0200548 orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
549 testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
550 jnz ia32_tracesys
551
Roland McGrathc09249f2009-02-06 18:15:18 -0800552ia32_do_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200553 /* 32-bit syscall -> 64-bit C ABI argument conversion */
554 movl %edi, %r8d /* arg5 */
555 movl %ebp, %r9d /* arg6 */
556 xchg %ecx, %esi /* rsi:arg2, rcx:arg4 */
557 movl %ebx, %edi /* arg1 */
558 movl %edx, %edx /* arg3 (zero extension) */
559 cmpq $(IA32_NR_syscalls-1), %rax
Denys Vlasenko3f5159a2015-04-21 18:03:14 +0200560 ja 1f
Ingo Molnar54ad7262015-06-05 13:02:28 +0200561
Denys Vlasenkoeb478542015-06-07 20:24:30 +0200562 call *ia32_sys_call_table(, %rax, 8)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200563 movq %rax, RAX(%rsp)
Denys Vlasenko3f5159a2015-04-21 18:03:14 +02005641:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200565 jmp int_ret_from_sys_call
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800567ia32_tracesys:
568 SAVE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200569 movq %rsp, %rdi /* &pt_regs -> arg1 */
570 call syscall_trace_enter
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200571 /*
572 * Reload arg registers from stack in case ptrace changed them.
573 * Don't reload %eax because syscall_trace_enter() returned
574 * the %rax value we should see. But do truncate it to 32 bits.
575 * If it's -1 to make us punt the syscall, then (u32)-1 is still
576 * an appropriately invalid value.
577 */
578 movl RCX(%rsp), %ecx
579 movl RDX(%rsp), %edx
580 movl RSI(%rsp), %esi
581 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200582 movl %eax, %eax /* zero extension */
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800583 RESTORE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200584 jmp ia32_do_call
Ingo Molnar2cd23552015-06-08 08:28:07 +0200585END(entry_INT80_compat)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586
Ramkumar Ramachandrad2475b82013-07-10 23:34:28 +0530587 .macro PTREGSCALL label, func
Jan Beulichf6b2bc82011-11-29 11:24:10 +0000588 ALIGN
589GLOBAL(\label)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200590 leaq \func(%rip), %rax
591 jmp ia32_ptregs_common
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 .endm
593
Ingo Molnar54ad7262015-06-05 13:02:28 +0200594 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn
595 PTREGSCALL stub32_sigreturn, sys32_sigreturn
596 PTREGSCALL stub32_fork, sys_fork
597 PTREGSCALL stub32_vfork, sys_vfork
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598
Jan Beulichf6b2bc82011-11-29 11:24:10 +0000599 ALIGN
Al Viro1d4b4b22012-10-22 22:34:11 -0400600GLOBAL(stub32_clone)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200601 leaq sys_clone(%rip), %rax
Denys Vlasenko5cdc6832015-06-03 15:58:49 +0200602 /*
Denys Vlasenko7a5a9822015-06-03 15:58:50 +0200603 * The 32-bit clone ABI is: clone(..., int tls_val, int *child_tidptr).
604 * The 64-bit clone ABI is: clone(..., int *child_tidptr, int tls_val).
605 *
606 * The native 64-bit kernel's sys_clone() implements the latter,
607 * so we need to swap arguments here before calling it:
Denys Vlasenko5cdc6832015-06-03 15:58:49 +0200608 */
Denys Vlasenko7a5a9822015-06-03 15:58:50 +0200609 xchg %r8, %rcx
Ingo Molnar54ad7262015-06-05 13:02:28 +0200610 jmp ia32_ptregs_common
Al Viro1d4b4b22012-10-22 22:34:11 -0400611
612 ALIGN
Jan Beulichf6b2bc82011-11-29 11:24:10 +0000613ia32_ptregs_common:
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800614 SAVE_EXTRA_REGS 8
Ingo Molnar54ad7262015-06-05 13:02:28 +0200615 call *%rax
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800616 RESTORE_EXTRA_REGS 8
617 ret
Jan Beulich4b787e02006-06-26 13:56:55 +0200618END(ia32_ptregs_common)