blob: e2cca89c1ed33a9f4b5ee00d37c18b39f9c10600 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Ingo Molnar54ad7262015-06-05 13:02:28 +02002 * Compatibility mode system call entry point for x86-64.
3 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
Ingo Molnar54ad7262015-06-05 13:02:28 +02005 */
Ingo Molnard36f9472015-06-03 18:29:26 +02006#include "calling.h"
Sam Ravnborge2d5df92005-09-09 21:28:48 +02007#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <asm/current.h>
9#include <asm/errno.h>
Ingo Molnar54ad7262015-06-05 13:02:28 +020010#include <asm/ia32_unistd.h>
11#include <asm/thread_info.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <asm/segment.h>
Ingo Molnar2601e642006-07-03 00:24:45 -070013#include <asm/irqflags.h>
H. Peter Anvin1ce6f862012-04-20 12:19:50 -070014#include <asm/asm.h>
H. Peter Anvin63bcff22012-09-21 12:43:12 -070015#include <asm/smap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/linkage.h>
Eric Parisd7e75282012-01-03 14:23:06 -050017#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
Roland McGrath5cbf1562008-06-24 01:13:31 -070019/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
20#include <linux/elf-em.h>
21#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
Ingo Molnar54ad7262015-06-05 13:02:28 +020022#define __AUDIT_ARCH_LE 0x40000000
Roland McGrath5cbf1562008-06-24 01:13:31 -070023
24#ifndef CONFIG_AUDITSYSCALL
Andy Lutomirski8f7f06b2015-07-07 10:55:28 -070025# define sysexit_audit ia32_ret_from_sys_call_irqs_off
26# define sysretl_audit ia32_ret_from_sys_call_irqs_off
Roland McGrath5cbf1562008-06-24 01:13:31 -070027#endif
28
Jiri Olsaea714542011-03-07 19:10:39 +010029 .section .entry.text, "ax"
30
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -040031#ifdef CONFIG_PARAVIRT
32ENTRY(native_usergs_sysret32)
33 swapgs
34 sysretl
35ENDPROC(native_usergs_sysret32)
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -040036#endif
37
Linus Torvalds1da177e2005-04-16 15:20:36 -070038/*
Ingo Molnar54ad7262015-06-05 13:02:28 +020039 * 32-bit SYSENTER instruction entry.
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 *
Denys Vlasenkob87cf632015-02-26 14:40:32 -080041 * SYSENTER loads ss, rsp, cs, and rip from previously programmed MSRs.
42 * IF and VM in rflags are cleared (IOW: interrupts are off).
43 * SYSENTER does not save anything on the stack,
44 * and does not save old rip (!!!) and rflags.
45 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * Arguments:
Denys Vlasenkob87cf632015-02-26 14:40:32 -080047 * eax system call number
48 * ebx arg1
49 * ecx arg2
50 * edx arg3
51 * esi arg4
52 * edi arg5
53 * ebp user stack
54 * 0(%ebp) arg6
55 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 * This is purely a fast path. For anything complicated we use the int 0x80
Denys Vlasenkob87cf632015-02-26 14:40:32 -080057 * path below. We set up a complete hardware stack frame to share code
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 * with the int 0x80 path.
Denys Vlasenkob87cf632015-02-26 14:40:32 -080059 */
Ingo Molnar4c8cd0c2015-06-08 08:33:56 +020060ENTRY(entry_SYSENTER_compat)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +010061 /*
62 * Interrupts are off on entry.
63 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
64 * it is too small to ever cause noticeable irq latency.
65 */
Jeremy Fitzhardinge457da702008-06-26 07:28:51 -070066 SWAPGS_UNSAFE_STACK
Denys Vlasenko3a232082015-04-24 17:31:35 +020067 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -040068 ENABLE_INTERRUPTS(CLBR_NONE)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +010069
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +010070 /* Zero-extending 32-bit regs, do not remove */
71 movl %ebp, %ebp
72 movl %eax, %eax
73
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020074 movl ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020075
76 /* Construct struct pt_regs on stack */
Ingo Molnar131484c2015-05-28 12:21:47 +020077 pushq $__USER32_DS /* pt_regs->ss */
78 pushq %rbp /* pt_regs->sp */
79 pushfq /* pt_regs->flags */
80 pushq $__USER32_CS /* pt_regs->cs */
Ingo Molnar54ad7262015-06-05 13:02:28 +020081 pushq %r10 /* pt_regs->ip = thread_info->sysenter_return */
Ingo Molnar131484c2015-05-28 12:21:47 +020082 pushq %rax /* pt_regs->orig_ax */
83 pushq %rdi /* pt_regs->di */
84 pushq %rsi /* pt_regs->si */
85 pushq %rdx /* pt_regs->dx */
86 pushq %rcx /* pt_regs->cx */
87 pushq $-ENOSYS /* pt_regs->ax */
Linus Torvalds1da177e2005-04-16 15:20:36 -070088 cld
Ingo Molnar54ad7262015-06-05 13:02:28 +020089 sub $(10*8), %rsp /* pt_regs->r8-11, bp, bx, r12-15 not saved */
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020090
Denys Vlasenkob87cf632015-02-26 14:40:32 -080091 /*
Andy Lutomirski8c7aa692014-10-01 11:49:04 -070092 * Sysenter doesn't filter flags, so we need to clear NT
93 * ourselves. To save a few cycles, we can check whether
94 * NT was set instead of doing an unconditional popfq.
95 */
Ingo Molnar54ad7262015-06-05 13:02:28 +020096 testl $X86_EFLAGS_NT, EFLAGS(%rsp)
97 jnz sysenter_fix_flags
Andy Lutomirski8c7aa692014-10-01 11:49:04 -070098sysenter_flags_fixed:
99
Andy Lutomirskidd27f992015-10-05 17:47:53 -0700100 /*
101 * No need to do an access_ok() check here because RBP has been
102 * 32-bit zero extended:
103 */
104 ASM_STAC
1051: movl (%rbp), %ebp
106 _ASM_EXTABLE(1b, ia32_badarg)
107 ASM_CLAC
108
Denys Vlasenko9b47feb2015-06-08 22:35:33 +0200109 orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
110 testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200111 jnz sysenter_tracesys
Ingo Molnar131484c2015-05-28 12:21:47 +0200112
Roland McGrathd4d67152008-07-09 02:38:07 -0700113sysenter_do_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200114 /* 32-bit syscall -> 64-bit C ABI argument conversion */
115 movl %edi, %r8d /* arg5 */
116 movl %ebp, %r9d /* arg6 */
117 xchg %ecx, %esi /* rsi:arg2, rcx:arg4 */
118 movl %ebx, %edi /* arg1 */
119 movl %edx, %edx /* arg3 (zero extension) */
Roland McGrath5cbf1562008-06-24 01:13:31 -0700120sysenter_dispatch:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200121 cmpq $(IA32_NR_syscalls-1), %rax
Denys Vlasenko3f5159a2015-04-21 18:03:14 +0200122 ja 1f
Ingo Molnar54ad7262015-06-05 13:02:28 +0200123 call *ia32_sys_call_table(, %rax, 8)
124 movq %rax, RAX(%rsp)
Denys Vlasenko3f5159a2015-04-21 18:03:14 +02001251:
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400126 DISABLE_INTERRUPTS(CLBR_NONE)
Ingo Molnar2601e642006-07-03 00:24:45 -0700127 TRACE_IRQS_OFF
Ingo Molnardca5b522015-03-24 19:44:42 +0100128 testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
Roland McGrath5cbf1562008-06-24 01:13:31 -0700129 jnz sysexit_audit
130sysexit_from_sys_call:
Andy Lutomirski4214a162015-04-02 17:12:12 -0700131 /*
132 * NB: SYSEXIT is not obviously safe for 64-bit kernels -- an
133 * NMI between STI and SYSEXIT has poorly specified behavior,
134 * and and NMI followed by an IRQ with usergs is fatal. So
135 * we just pretend we're using SYSEXIT but we really use
136 * SYSRETL instead.
137 *
138 * This code path is still called 'sysexit' because it pairs
139 * with 'sysenter' and it uses the SYSENTER calling convention.
140 */
Denys Vlasenko9b47feb2015-06-08 22:35:33 +0200141 andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200142 movl RIP(%rsp), %ecx /* User %eip */
Linus Torvaldscd88ec22015-08-13 16:19:44 -0700143 movq RAX(%rsp), %rax
Denys Vlasenkoc73e36b2015-07-03 22:19:02 +0200144 movl RSI(%rsp), %esi
145 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200146 xorl %edx, %edx /* Do not leak kernel information */
147 xorq %r8, %r8
148 xorq %r9, %r9
149 xorq %r10, %r10
150 movl EFLAGS(%rsp), %r11d /* User eflags */
Ingo Molnar2601e642006-07-03 00:24:45 -0700151 TRACE_IRQS_ON
Andy Lutomirski4214a162015-04-02 17:12:12 -0700152
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800153 /*
Andy Lutomirski4214a162015-04-02 17:12:12 -0700154 * SYSRETL works even on Intel CPUs. Use it in preference to SYSEXIT,
155 * since it avoids a dicey window with interrupts enabled.
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800156 */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200157 movl RSP(%rsp), %esp
Andy Lutomirski4214a162015-04-02 17:12:12 -0700158
159 /*
160 * USERGS_SYSRET32 does:
161 * gsbase = user's gs base
162 * eip = ecx
163 * rflags = r11
164 * cs = __USER32_CS
165 * ss = __USER_DS
166 *
167 * The prologue set RIP(%rsp) to VDSO32_SYSENTER_RETURN, which does:
168 *
169 * pop %ebp
170 * pop %edx
171 * pop %ecx
172 *
173 * Therefore, we invoke SYSRETL with EDX and R8-R10 zeroed to
174 * avoid info leaks. R11 ends up with VDSO32_SYSENTER_RETURN's
175 * address (already known to user code), and R12-R15 are
176 * callee-saved and therefore don't contain any interesting
177 * kernel data.
178 */
179 USERGS_SYSRET32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180
Roland McGrath5cbf1562008-06-24 01:13:31 -0700181#ifdef CONFIG_AUDITSYSCALL
182 .macro auditsys_entry_common
Denys Vlasenko1536bb42015-06-09 20:54:08 +0200183 /*
Denys Vlasenkoa92fde252015-06-09 20:54:09 +0200184 * At this point, registers hold syscall args in the 32-bit syscall ABI:
185 * EAX is syscall number, the 6 args are in EBX,ECX,EDX,ESI,EDI,EBP.
186 *
187 * We want to pass them to __audit_syscall_entry(), which is a 64-bit
188 * C function with 5 parameters, so shuffle them to match what
189 * the function expects: RDI,RSI,RDX,RCX,R8.
190 */
191 movl %esi, %r8d /* arg5 (R8 ) <= 4th syscall arg (ESI) */
192 xchg %ecx, %edx /* arg4 (RCX) <= 3rd syscall arg (EDX) */
193 /* arg3 (RDX) <= 2nd syscall arg (ECX) */
194 movl %ebx, %esi /* arg2 (RSI) <= 1st syscall arg (EBX) */
195 movl %eax, %edi /* arg1 (RDI) <= syscall number (EAX) */
196 call __audit_syscall_entry
197
198 /*
199 * We are going to jump back to the syscall dispatch code.
200 * Prepare syscall args as required by the 64-bit C ABI.
201 * Registers clobbered by __audit_syscall_entry() are
202 * loaded from pt_regs on stack:
Denys Vlasenko1536bb42015-06-09 20:54:08 +0200203 */
204 movl ORIG_RAX(%rsp), %eax /* syscall number */
205 movl %ebx, %edi /* arg1 */
206 movl RCX(%rsp), %esi /* arg2 */
207 movl RDX(%rsp), %edx /* arg3 */
208 movl RSI(%rsp), %ecx /* arg4 */
209 movl RDI(%rsp), %r8d /* arg5 */
Roland McGrath5cbf1562008-06-24 01:13:31 -0700210 .endm
211
Jan Beulich81766742009-10-26 15:20:29 +0000212 .macro auditsys_exit exit
Roland McGrath5cbf1562008-06-24 01:13:31 -0700213 TRACE_IRQS_ON
Jan Beulich40a1ef92013-01-30 07:55:53 +0000214 ENABLE_INTERRUPTS(CLBR_NONE)
Andy Lutomirski29ea1b22015-07-03 12:44:28 -0700215 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
216 jnz ia32_ret_from_sys_call
Ingo Molnar54ad7262015-06-05 13:02:28 +0200217 movl %eax, %esi /* second arg, syscall return value */
218 cmpl $-MAX_ERRNO, %eax /* is it an error ? */
219 jbe 1f
220 movslq %eax, %rsi /* if error sign extend to 64 bits */
2211: setbe %al /* 1 if error, 0 if not */
222 movzbl %al, %edi /* zero-extend that into %edi */
223 call __audit_syscall_exit
Ingo Molnar54ad7262015-06-05 13:02:28 +0200224 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %edi
Jan Beulich40a1ef92013-01-30 07:55:53 +0000225 DISABLE_INTERRUPTS(CLBR_NONE)
Roland McGrath5cbf1562008-06-24 01:13:31 -0700226 TRACE_IRQS_OFF
Ingo Molnar54ad7262015-06-05 13:02:28 +0200227 testl %edi, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
228 jz \exit
229 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200230 movq %rax, R11(%rsp)
231 movq %rax, R10(%rsp)
232 movq %rax, R9(%rsp)
233 movq %rax, R8(%rsp)
Andy Lutomirski29ea1b22015-07-03 12:44:28 -0700234 jmp int_ret_from_sys_call_irqs_off
Roland McGrath5cbf1562008-06-24 01:13:31 -0700235 .endm
236
237sysenter_auditsys:
Roland McGrath5cbf1562008-06-24 01:13:31 -0700238 auditsys_entry_common
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200239 movl %ebp, %r9d /* reload 6th syscall arg */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200240 jmp sysenter_dispatch
Roland McGrath5cbf1562008-06-24 01:13:31 -0700241
242sysexit_audit:
243 auditsys_exit sysexit_from_sys_call
244#endif
245
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700246sysenter_fix_flags:
Denys Vlasenko9b47feb2015-06-08 22:35:33 +0200247 pushq $(X86_EFLAGS_IF|X86_EFLAGS_FIXED)
Ingo Molnar131484c2015-05-28 12:21:47 +0200248 popfq
Ingo Molnar54ad7262015-06-05 13:02:28 +0200249 jmp sysenter_flags_fixed
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700250
Roland McGrath5cbf1562008-06-24 01:13:31 -0700251sysenter_tracesys:
Roland McGrath5cbf1562008-06-24 01:13:31 -0700252#ifdef CONFIG_AUDITSYSCALL
Ingo Molnardca5b522015-03-24 19:44:42 +0100253 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
Roland McGrath5cbf1562008-06-24 01:13:31 -0700254 jz sysenter_auditsys
255#endif
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800256 SAVE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200257 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200258 movq %rax, R11(%rsp)
259 movq %rax, R10(%rsp)
260 movq %rax, R9(%rsp)
261 movq %rax, R8(%rsp)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200262 movq %rsp, %rdi /* &pt_regs -> arg1 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 call syscall_trace_enter
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200264
265 /* Reload arg registers from stack. (see sysenter_tracesys) */
266 movl RCX(%rsp), %ecx
267 movl RDX(%rsp), %edx
268 movl RSI(%rsp), %esi
269 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200270 movl %eax, %eax /* zero extension */
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200271
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800272 RESTORE_EXTRA_REGS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 jmp sysenter_do_call
Ingo Molnar4c8cd0c2015-06-08 08:33:56 +0200274ENDPROC(entry_SYSENTER_compat)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275
276/*
Ingo Molnar54ad7262015-06-05 13:02:28 +0200277 * 32-bit SYSCALL instruction entry.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 *
Ingo Molnar54ad7262015-06-05 13:02:28 +0200279 * 32-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800280 * then loads new ss, cs, and rip from previously programmed MSRs.
281 * rflags gets masked by a value from another MSR (so CLD and CLAC
282 * are not needed). SYSCALL does not save anything on the stack
283 * and does not change rsp.
284 *
285 * Note: rflags saving+masking-with-MSR happens only in Long mode
Ingo Molnar54ad7262015-06-05 13:02:28 +0200286 * (in legacy 32-bit mode, IF, RF and VM bits are cleared and that's it).
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800287 * Don't get confused: rflags saving+masking depends on Long Mode Active bit
288 * (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
289 * or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
290 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 * Arguments:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800292 * eax system call number
293 * ecx return address
294 * ebx arg1
295 * ebp arg2 (note: not saved in the stack frame, should not be touched)
296 * edx arg3
297 * esi arg4
298 * edi arg5
299 * esp user stack
300 * 0(%esp) arg6
301 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 * This is purely a fast path. For anything complicated we use the int 0x80
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800303 * path below. We set up a complete hardware stack frame to share code
304 * with the int 0x80 path.
305 */
Ingo Molnar2cd23552015-06-08 08:28:07 +0200306ENTRY(entry_SYSCALL_compat)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100307 /*
308 * Interrupts are off on entry.
309 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
310 * it is too small to ever cause noticeable irq latency.
311 */
Jeremy Fitzhardinge457da702008-06-26 07:28:51 -0700312 SWAPGS_UNSAFE_STACK
Ingo Molnar54ad7262015-06-05 13:02:28 +0200313 movl %esp, %r8d
314 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400315 ENABLE_INTERRUPTS(CLBR_NONE)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100316
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100317 /* Zero-extending 32-bit regs, do not remove */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200318 movl %eax, %eax
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100319
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200320 /* Construct struct pt_regs on stack */
Ingo Molnar131484c2015-05-28 12:21:47 +0200321 pushq $__USER32_DS /* pt_regs->ss */
322 pushq %r8 /* pt_regs->sp */
323 pushq %r11 /* pt_regs->flags */
324 pushq $__USER32_CS /* pt_regs->cs */
325 pushq %rcx /* pt_regs->ip */
326 pushq %rax /* pt_regs->orig_ax */
327 pushq %rdi /* pt_regs->di */
328 pushq %rsi /* pt_regs->si */
329 pushq %rdx /* pt_regs->dx */
330 pushq %rbp /* pt_regs->cx */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200331 movl %ebp, %ecx
Ingo Molnar131484c2015-05-28 12:21:47 +0200332 pushq $-ENOSYS /* pt_regs->ax */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200333 sub $(10*8), %rsp /* pt_regs->r8-11, bp, bx, r12-15 not saved */
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200334
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800335 /*
Ingo Molnar54ad7262015-06-05 13:02:28 +0200336 * No need to do an access_ok check here because r8 has been
337 * 32-bit zero extended:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800338 */
H. Peter Anvin63bcff22012-09-21 12:43:12 -0700339 ASM_STAC
Denys Vlasenkoc0c33222015-07-24 14:16:43 +02003401: movl (%r8), %r9d
Ingo Molnar54ad7262015-06-05 13:02:28 +0200341 _ASM_EXTABLE(1b, ia32_badarg)
H. Peter Anvin63bcff22012-09-21 12:43:12 -0700342 ASM_CLAC
Denys Vlasenko9b47feb2015-06-08 22:35:33 +0200343 orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
344 testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
345 jnz cstar_tracesys
Ingo Molnar131484c2015-05-28 12:21:47 +0200346
Jan Beulich295286a82008-08-29 13:21:11 +0100347cstar_do_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200348 /* 32-bit syscall -> 64-bit C ABI argument conversion */
349 movl %edi, %r8d /* arg5 */
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200350 /* r9 already loaded */ /* arg6 */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200351 xchg %ecx, %esi /* rsi:arg2, rcx:arg4 */
352 movl %ebx, %edi /* arg1 */
353 movl %edx, %edx /* arg3 (zero extension) */
354
Roland McGrath5cbf1562008-06-24 01:13:31 -0700355cstar_dispatch:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200356 cmpq $(IA32_NR_syscalls-1), %rax
Denys Vlasenko3f5159a2015-04-21 18:03:14 +0200357 ja 1f
Ingo Molnar54ad7262015-06-05 13:02:28 +0200358
359 call *ia32_sys_call_table(, %rax, 8)
360 movq %rax, RAX(%rsp)
Denys Vlasenko3f5159a2015-04-21 18:03:14 +02003611:
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400362 DISABLE_INTERRUPTS(CLBR_NONE)
Ingo Molnar2601e642006-07-03 00:24:45 -0700363 TRACE_IRQS_OFF
Ingo Molnar54ad7262015-06-05 13:02:28 +0200364 testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
365 jnz sysretl_audit
366
Roland McGrath5cbf1562008-06-24 01:13:31 -0700367sysretl_from_sys_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200368 andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
Denys Vlasenkoc73e36b2015-07-03 22:19:02 +0200369 movl RDX(%rsp), %edx
370 movl RSI(%rsp), %esi
371 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200372 movl RIP(%rsp), %ecx
373 movl EFLAGS(%rsp), %r11d
Linus Torvaldscd88ec22015-08-13 16:19:44 -0700374 movq RAX(%rsp), %rax
Ingo Molnar54ad7262015-06-05 13:02:28 +0200375 xorq %r10, %r10
376 xorq %r9, %r9
377 xorq %r8, %r8
Ingo Molnar2601e642006-07-03 00:24:45 -0700378 TRACE_IRQS_ON
Ingo Molnar54ad7262015-06-05 13:02:28 +0200379 movl RSP(%rsp), %esp
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800380 /*
Ingo Molnar54ad7262015-06-05 13:02:28 +0200381 * 64-bit->32-bit SYSRET restores eip from ecx,
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800382 * eflags from r11 (but RF and VM bits are forced to 0),
383 * cs and ss are loaded from MSRs.
Ingo Molnar54ad7262015-06-05 13:02:28 +0200384 * (Note: 32-bit->32-bit SYSRET is different: since r11
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800385 * does not exist, it merely sets eflags.IF=1).
Andy Lutomirski61f01dd2015-04-26 16:47:59 -0700386 *
387 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
388 * descriptor is not reinitialized. This means that we must
389 * avoid SYSRET with SS == NULL, which could happen if we schedule,
390 * exit the kernel, and re-enter using an interrupt vector. (All
391 * interrupt entries on x86_64 set SS to NULL.) We prevent that
392 * from happening by reloading SS in __switch_to.
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800393 */
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -0400394 USERGS_SYSRET32
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800395
Roland McGrath5cbf1562008-06-24 01:13:31 -0700396#ifdef CONFIG_AUDITSYSCALL
397cstar_auditsys:
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200398 movl %r9d, R9(%rsp) /* register to be clobbered by call */
Roland McGrath5cbf1562008-06-24 01:13:31 -0700399 auditsys_entry_common
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200400 movl R9(%rsp), %r9d /* reload 6th syscall arg */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200401 jmp cstar_dispatch
Roland McGrath5cbf1562008-06-24 01:13:31 -0700402
403sysretl_audit:
Jan Beulich81766742009-10-26 15:20:29 +0000404 auditsys_exit sysretl_from_sys_call
Roland McGrath5cbf1562008-06-24 01:13:31 -0700405#endif
406
407cstar_tracesys:
408#ifdef CONFIG_AUDITSYSCALL
Ingo Molnar54ad7262015-06-05 13:02:28 +0200409 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
410 jz cstar_auditsys
Roland McGrath5cbf1562008-06-24 01:13:31 -0700411#endif
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200412 xchgl %r9d, %ebp
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800413 SAVE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200414 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200415 movq %rax, R11(%rsp)
416 movq %rax, R10(%rsp)
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200417 movq %r9, R9(%rsp)
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200418 movq %rax, R8(%rsp)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200419 movq %rsp, %rdi /* &pt_regs -> arg1 */
Denys Vlasenko53e9acc2015-06-03 14:56:09 +0200420 call syscall_trace_enter
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200421 movl R9(%rsp), %r9d
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200422
423 /* Reload arg registers from stack. (see sysenter_tracesys) */
424 movl RCX(%rsp), %ecx
425 movl RDX(%rsp), %edx
426 movl RSI(%rsp), %esi
427 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200428 movl %eax, %eax /* zero extension */
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200429
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800430 RESTORE_EXTRA_REGS
Denys Vlasenkoc0c33222015-07-24 14:16:43 +0200431 xchgl %ebp, %r9d
Denys Vlasenko53e9acc2015-06-03 14:56:09 +0200432 jmp cstar_do_call
Ingo Molnar2cd23552015-06-08 08:28:07 +0200433END(entry_SYSCALL_compat)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200434
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435ia32_badarg:
Andy Lutomirski5e99cb72015-07-03 12:44:19 -0700436 /*
437 * So far, we've entered kernel mode, set AC, turned on IRQs, and
438 * saved C regs except r8-r11. We haven't done any of the other
439 * standard entry work, though. We want to bail, but we shouldn't
440 * treat this as a syscall entry since we don't even know what the
441 * args are. Instead, treat this as a non-syscall entry, finish
442 * the entry work, and immediately exit after setting AX = -EFAULT.
443 *
444 * We're really just being polite here. Killing the task outright
445 * would be a reasonable action, too. Given that the only valid
446 * way to have gotten here is through the vDSO, and we already know
447 * that the stack pointer is bad, the task isn't going to survive
448 * for long no matter what we do.
449 */
450
451 ASM_CLAC /* undo STAC */
452 movq $-EFAULT, RAX(%rsp) /* return -EFAULT if possible */
453
454 /* Fill in the rest of pt_regs */
455 xorl %eax, %eax
456 movq %rax, R11(%rsp)
457 movq %rax, R10(%rsp)
458 movq %rax, R9(%rsp)
459 movq %rax, R8(%rsp)
460 SAVE_EXTRA_REGS
461
462 /* Turn IRQs back off. */
463 DISABLE_INTERRUPTS(CLBR_NONE)
464 TRACE_IRQS_OFF
465
Andy Lutomirski02bc7762015-07-03 12:44:31 -0700466 /* Now finish entering normal kernel mode. */
467#ifdef CONFIG_CONTEXT_TRACKING
468 call enter_from_user_mode
469#endif
470
Andy Lutomirski5e99cb72015-07-03 12:44:19 -0700471 /* And exit again. */
472 jmp retint_user
473
Andy Lutomirski8f7f06b2015-07-07 10:55:28 -0700474ia32_ret_from_sys_call_irqs_off:
475 TRACE_IRQS_ON
476 ENABLE_INTERRUPTS(CLBR_NONE)
477
Denys Vlasenko61b1e3e2015-06-02 19:35:10 +0200478ia32_ret_from_sys_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200479 xorl %eax, %eax /* Do not leak kernel information */
Denys Vlasenkoef0cd5d2015-06-02 21:04:01 +0200480 movq %rax, R11(%rsp)
481 movq %rax, R10(%rsp)
482 movq %rax, R9(%rsp)
483 movq %rax, R8(%rsp)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200484 jmp int_ret_from_sys_call
Denys Vlasenko61b1e3e2015-06-02 19:35:10 +0200485
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800486/*
487 * Emulated IA32 system calls via int 0x80.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 *
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800489 * Arguments:
490 * eax system call number
491 * ebx arg1
492 * ecx arg2
493 * edx arg3
494 * esi arg4
495 * edi arg5
496 * ebp arg6 (note: not saved in the stack frame, should not be touched)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 *
498 * Notes:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800499 * Uses the same stack frame as the x86-64 version.
500 * All registers except eax must be saved (but ptrace may violate that).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 * Arguments are zero extended. For system calls that want sign extension and
502 * take long arguments a wrapper is needed. Most calls can just be called
503 * directly.
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800504 * Assumes it is only called from user space and entered with interrupts off.
505 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506
Ingo Molnar2cd23552015-06-08 08:28:07 +0200507ENTRY(entry_INT80_compat)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100508 /*
509 * Interrupts are off on entry.
510 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
511 * it is too small to ever cause noticeable irq latency.
512 */
Jeremy Fitzhardinge360c0442008-07-08 15:06:28 -0700513 PARAVIRT_ADJUST_EXCEPTION_FRAME
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400514 SWAPGS
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400515 ENABLE_INTERRUPTS(CLBR_NONE)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100516
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100517 /* Zero-extending 32-bit regs, do not remove */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200518 movl %eax, %eax
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100519
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200520 /* Construct struct pt_regs on stack (iret frame is already on stack) */
Ingo Molnar131484c2015-05-28 12:21:47 +0200521 pushq %rax /* pt_regs->orig_ax */
522 pushq %rdi /* pt_regs->di */
523 pushq %rsi /* pt_regs->si */
524 pushq %rdx /* pt_regs->dx */
525 pushq %rcx /* pt_regs->cx */
526 pushq $-ENOSYS /* pt_regs->ax */
Denys Vlasenko61b1e3e2015-06-02 19:35:10 +0200527 pushq $0 /* pt_regs->r8 */
528 pushq $0 /* pt_regs->r9 */
529 pushq $0 /* pt_regs->r10 */
530 pushq $0 /* pt_regs->r11 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 cld
Ingo Molnar54ad7262015-06-05 13:02:28 +0200532 sub $(6*8), %rsp /* pt_regs->bp, bx, r12-15 not saved */
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200533
Ingo Molnar54ad7262015-06-05 13:02:28 +0200534 orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
535 testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
536 jnz ia32_tracesys
537
Roland McGrathc09249f2009-02-06 18:15:18 -0800538ia32_do_call:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200539 /* 32-bit syscall -> 64-bit C ABI argument conversion */
540 movl %edi, %r8d /* arg5 */
541 movl %ebp, %r9d /* arg6 */
542 xchg %ecx, %esi /* rsi:arg2, rcx:arg4 */
543 movl %ebx, %edi /* arg1 */
544 movl %edx, %edx /* arg3 (zero extension) */
545 cmpq $(IA32_NR_syscalls-1), %rax
Denys Vlasenko3f5159a2015-04-21 18:03:14 +0200546 ja 1f
Ingo Molnar54ad7262015-06-05 13:02:28 +0200547
Denys Vlasenkoeb478542015-06-07 20:24:30 +0200548 call *ia32_sys_call_table(, %rax, 8)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200549 movq %rax, RAX(%rsp)
Denys Vlasenko3f5159a2015-04-21 18:03:14 +02005501:
Ingo Molnar54ad7262015-06-05 13:02:28 +0200551 jmp int_ret_from_sys_call
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800553ia32_tracesys:
554 SAVE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200555 movq %rsp, %rdi /* &pt_regs -> arg1 */
556 call syscall_trace_enter
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200557 /*
558 * Reload arg registers from stack in case ptrace changed them.
559 * Don't reload %eax because syscall_trace_enter() returned
560 * the %rax value we should see. But do truncate it to 32 bits.
561 * If it's -1 to make us punt the syscall, then (u32)-1 is still
562 * an appropriately invalid value.
563 */
564 movl RCX(%rsp), %ecx
565 movl RDX(%rsp), %edx
566 movl RSI(%rsp), %esi
567 movl RDI(%rsp), %edi
Ingo Molnar54ad7262015-06-05 13:02:28 +0200568 movl %eax, %eax /* zero extension */
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800569 RESTORE_EXTRA_REGS
Ingo Molnar54ad7262015-06-05 13:02:28 +0200570 jmp ia32_do_call
Ingo Molnar2cd23552015-06-08 08:28:07 +0200571END(entry_INT80_compat)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572
Ramkumar Ramachandrad2475b82013-07-10 23:34:28 +0530573 .macro PTREGSCALL label, func
Jan Beulichf6b2bc82011-11-29 11:24:10 +0000574 ALIGN
575GLOBAL(\label)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200576 leaq \func(%rip), %rax
577 jmp ia32_ptregs_common
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 .endm
579
Ingo Molnar54ad7262015-06-05 13:02:28 +0200580 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn
581 PTREGSCALL stub32_sigreturn, sys32_sigreturn
582 PTREGSCALL stub32_fork, sys_fork
583 PTREGSCALL stub32_vfork, sys_vfork
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584
Jan Beulichf6b2bc82011-11-29 11:24:10 +0000585 ALIGN
Al Viro1d4b4b22012-10-22 22:34:11 -0400586GLOBAL(stub32_clone)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200587 leaq sys_clone(%rip), %rax
Denys Vlasenko5cdc6832015-06-03 15:58:49 +0200588 /*
Denys Vlasenko7a5a9822015-06-03 15:58:50 +0200589 * The 32-bit clone ABI is: clone(..., int tls_val, int *child_tidptr).
590 * The 64-bit clone ABI is: clone(..., int *child_tidptr, int tls_val).
591 *
592 * The native 64-bit kernel's sys_clone() implements the latter,
593 * so we need to swap arguments here before calling it:
Denys Vlasenko5cdc6832015-06-03 15:58:49 +0200594 */
Denys Vlasenko7a5a9822015-06-03 15:58:50 +0200595 xchg %r8, %rcx
Ingo Molnar54ad7262015-06-05 13:02:28 +0200596 jmp ia32_ptregs_common
Al Viro1d4b4b22012-10-22 22:34:11 -0400597
598 ALIGN
Jan Beulichf6b2bc82011-11-29 11:24:10 +0000599ia32_ptregs_common:
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800600 SAVE_EXTRA_REGS 8
Ingo Molnar54ad7262015-06-05 13:02:28 +0200601 call *%rax
Denys Vlasenko76f5df42015-02-26 14:40:27 -0800602 RESTORE_EXTRA_REGS 8
603 ret
Jan Beulich4b787e02006-06-26 13:56:55 +0200604END(ia32_ptregs_common)