blob: 00348980a3a64a49180be23bda3517d314c6bf81 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * (C) Copyright 2002 Linus Torvalds
Ingo Molnare6e54942006-06-27 02:53:50 -07003 * Portions based on the vdso-randomization code from exec-shield:
4 * Copyright(C) 2005-2006, Red Hat, Inc., Ingo Molnar
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * This file contains the needed initializations to support sysenter.
7 */
8
9#include <linux/init.h>
10#include <linux/smp.h>
11#include <linux/thread_info.h>
12#include <linux/sched.h>
13#include <linux/gfp.h>
14#include <linux/string.h>
15#include <linux/elf.h>
Ingo Molnare6e54942006-06-27 02:53:50 -070016#include <linux/mm.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040017#include <linux/err.h>
Ingo Molnare6e54942006-06-27 02:53:50 -070018#include <linux/module.h>
Stefani Seibold4e401122014-03-17 23:22:13 +010019#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21#include <asm/cpufeature.h>
22#include <asm/msr.h>
23#include <asm/pgtable.h>
24#include <asm/unistd.h>
Jeremy Fitzhardinged4f7a2c2007-05-02 19:27:12 +020025#include <asm/elf.h>
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +020026#include <asm/tlbflush.h>
Roland McGrath6c3652e2008-01-30 13:30:42 +010027#include <asm/vdso.h>
Roland McGrathaf65d642008-01-30 13:30:43 +010028#include <asm/proto.h>
Stefani Seibold7a59ed42014-03-17 23:22:09 +010029#include <asm/fixmap.h>
30#include <asm/hpet.h>
31#include <asm/vvar.h>
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +020032
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +020033#ifdef CONFIG_COMPAT_VDSO
Andy Lutomirskib0b49f22014-03-13 16:01:26 -070034#define VDSO_DEFAULT 0
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +020035#else
Andy Lutomirskib0b49f22014-03-13 16:01:26 -070036#define VDSO_DEFAULT 1
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +020037#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
Roland McGrathaf65d642008-01-30 13:30:43 +010039#ifdef CONFIG_X86_64
40#define vdso_enabled sysctl_vsyscall32
41#define arch_setup_additional_pages syscall32_setup_pages
42#endif
43
44/*
Ingo Molnare6e54942006-06-27 02:53:50 -070045 * Should the kernel map a VDSO page into processes and pass its
46 * address down to glibc upon exec()?
47 */
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +020048unsigned int __read_mostly vdso_enabled = VDSO_DEFAULT;
Ingo Molnare6e54942006-06-27 02:53:50 -070049
Ingo Molnare6e54942006-06-27 02:53:50 -070050static int __init vdso_setup(char *s)
51{
52 vdso_enabled = simple_strtoul(s, NULL, 0);
53
Andy Lutomirskib0b49f22014-03-13 16:01:26 -070054 if (vdso_enabled > 1)
55 pr_warn("vdso32 values other than 0 and 1 are no longer allowed; vdso disabled\n");
56
Ingo Molnare6e54942006-06-27 02:53:50 -070057 return 1;
58}
59
Roland McGrathaf65d642008-01-30 13:30:43 +010060/*
61 * For consistency, the argument vdso32=[012] affects the 32-bit vDSO
62 * behavior on both 64-bit and 32-bit kernels.
63 * On 32-bit kernels, vdso=[012] means the same thing.
64 */
65__setup("vdso32=", vdso_setup);
Ingo Molnare6e54942006-06-27 02:53:50 -070066
Roland McGrathaf65d642008-01-30 13:30:43 +010067#ifdef CONFIG_X86_32
68__setup_param("vdso=", vdso32_setup, vdso_setup, 0);
69
70EXPORT_SYMBOL_GPL(vdso_enabled);
71#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
Stefani Seibold4e401122014-03-17 23:22:13 +010073static struct page **vdso32_pages;
Andy Lutomirskib67e6122014-03-20 15:01:21 -070074static unsigned vdso32_size;
Roland McGrathaf65d642008-01-30 13:30:43 +010075
76#ifdef CONFIG_X86_64
77
Jeremy Fitzhardingeb6ad92d2008-07-10 18:13:36 -070078#define vdso32_sysenter() (boot_cpu_has(X86_FEATURE_SYSENTER32))
Jeremy Fitzhardinge6a52e4b2008-07-12 02:22:00 -070079#define vdso32_syscall() (boot_cpu_has(X86_FEATURE_SYSCALL32))
Roland McGrathaf65d642008-01-30 13:30:43 +010080
81/* May not be __init: called during resume */
82void syscall32_cpu_init(void)
83{
Roland McGrathaf65d642008-01-30 13:30:43 +010084 /* Load these always in case some future AMD CPU supports
85 SYSENTER from compat mode too. */
H. Peter Anvin715c85b2012-06-07 13:32:04 -070086 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
87 wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
88 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)ia32_sysenter_target);
Roland McGrathaf65d642008-01-30 13:30:43 +010089
90 wrmsrl(MSR_CSTAR, ia32_cstar_target);
91}
92
Roland McGrathaf65d642008-01-30 13:30:43 +010093#else /* CONFIG_X86_32 */
94
95#define vdso32_sysenter() (boot_cpu_has(X86_FEATURE_SEP))
Jeremy Fitzhardinge6a52e4b2008-07-12 02:22:00 -070096#define vdso32_syscall() (0)
Roland McGrathaf65d642008-01-30 13:30:43 +010097
Li Shaohua6fe940d2005-06-25 14:54:53 -070098void enable_sep_cpu(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070099{
100 int cpu = get_cpu();
101 struct tss_struct *tss = &per_cpu(init_tss, cpu);
102
Li Shaohua6fe940d2005-06-25 14:54:53 -0700103 if (!boot_cpu_has(X86_FEATURE_SEP)) {
104 put_cpu();
105 return;
106 }
107
Rusty Russella75c54f2007-05-02 19:27:13 +0200108 tss->x86_tss.ss1 = __KERNEL_CS;
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100109 tss->x86_tss.sp1 = sizeof(struct tss_struct) + (unsigned long) tss;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0);
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100111 wrmsr(MSR_IA32_SYSENTER_ESP, tss->x86_tss.sp1, 0);
Roland McGrath0aa97fb222008-01-30 13:30:43 +0100112 wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long) ia32_sysenter_target, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 put_cpu();
114}
115
Roland McGrathaf65d642008-01-30 13:30:43 +0100116#endif /* CONFIG_X86_64 */
117
Jeremy Fitzhardingea6c4e072007-05-02 19:27:12 +0200118int __init sysenter_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119{
Andy Lutomirskib67e6122014-03-20 15:01:21 -0700120 char *vdso32_start, *vdso32_end;
121 int npages, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122
Andy Lutomirskib67e6122014-03-20 15:01:21 -0700123#ifdef CONFIG_COMPAT
Jeremy Fitzhardinge6a52e4b2008-07-12 02:22:00 -0700124 if (vdso32_syscall()) {
Andy Lutomirskib67e6122014-03-20 15:01:21 -0700125 vdso32_start = vdso32_syscall_start;
126 vdso32_end = vdso32_syscall_end;
127 vdso32_pages = vdso32_syscall_pages;
128 } else
129#endif
130 if (vdso32_sysenter()) {
131 vdso32_start = vdso32_sysenter_start;
132 vdso32_end = vdso32_sysenter_end;
133 vdso32_pages = vdso32_sysenter_pages;
Jeremy Fitzhardinge6a52e4b2008-07-12 02:22:00 -0700134 } else {
Andy Lutomirskib67e6122014-03-20 15:01:21 -0700135 vdso32_start = vdso32_int80_start;
136 vdso32_end = vdso32_int80_end;
137 vdso32_pages = vdso32_int80_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 }
139
Andy Lutomirskib67e6122014-03-20 15:01:21 -0700140 npages = ((vdso32_end - vdso32_start) + PAGE_SIZE - 1) / PAGE_SIZE;
141 vdso32_size = npages << PAGE_SHIFT;
142 for (i = 0; i < npages; i++)
143 vdso32_pages[i] = virt_to_page(vdso32_start + i*PAGE_SIZE);
Stefani Seibold4e401122014-03-17 23:22:13 +0100144
Andy Lutomirskib67e6122014-03-20 15:01:21 -0700145 patch_vdso32(vdso32_start, vdso32_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 return 0;
148}
Ingo Molnare6e54942006-06-27 02:53:50 -0700149
Ingo Molnare6e54942006-06-27 02:53:50 -0700150/* Setup a VMA at program startup for the vsyscall page */
Martin Schwidefskyfc5243d2008-12-25 13:38:35 +0100151int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
Ingo Molnare6e54942006-06-27 02:53:50 -0700152{
Ingo Molnare6e54942006-06-27 02:53:50 -0700153 struct mm_struct *mm = current->mm;
154 unsigned long addr;
Zachary Amsden752783c2007-05-02 19:27:16 +0200155 int ret = 0;
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100156 struct vm_area_struct *vma;
Ingo Molnare6e54942006-06-27 02:53:50 -0700157
H. J. Lu1a21d4e2012-02-19 11:38:06 -0800158#ifdef CONFIG_X86_X32_ABI
H. J. Lu1a21d4e2012-02-19 11:38:06 -0800159 if (test_thread_flag(TIF_X32))
H. Peter Anvin22e842d2012-02-21 14:32:19 -0800160 return x32_setup_additional_pages(bprm, uses_interp);
H. J. Lu1a21d4e2012-02-19 11:38:06 -0800161#endif
162
Andy Lutomirskib0b49f22014-03-13 16:01:26 -0700163 if (vdso_enabled != 1) /* Other values all mean "disabled" */
Roland McGrath5de253c2008-04-09 01:30:06 -0700164 return 0;
165
Ingo Molnare6e54942006-06-27 02:53:50 -0700166 down_write(&mm->mmap_sem);
Ingo Molnare6e54942006-06-27 02:53:50 -0700167
Stefani Seibold645a3872014-03-23 17:38:14 +0100168 addr = get_unmapped_area(NULL, 0, vdso32_size + VDSO_OFFSET(VDSO_PREV_PAGES), 0, 0);
Andy Lutomirskib0b49f22014-03-13 16:01:26 -0700169 if (IS_ERR_VALUE(addr)) {
170 ret = addr;
171 goto up_fail;
Roland McGrathaf65d642008-01-30 13:30:43 +0100172 }
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +0200173
Stefani Seibold645a3872014-03-23 17:38:14 +0100174 addr += VDSO_OFFSET(VDSO_PREV_PAGES);
175
Peter Zijlstraf7b6eb32009-06-05 14:04:51 +0200176 current->mm->context.vdso = (void *)addr;
177
Andy Lutomirskib0b49f22014-03-13 16:01:26 -0700178 /*
179 * MAYWRITE to allow gdb to COW and set breakpoints
180 */
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100181 ret = install_special_mapping(mm,
182 addr,
Andy Lutomirskib67e6122014-03-20 15:01:21 -0700183 vdso32_size,
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100184 VM_READ|VM_EXEC|
185 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
186 vdso32_pages);
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +0200187
Andy Lutomirskib0b49f22014-03-13 16:01:26 -0700188 if (ret)
189 goto up_fail;
Ingo Molnare6e54942006-06-27 02:53:50 -0700190
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100191 vma = _install_special_mapping(mm,
192 addr - VDSO_OFFSET(VDSO_PREV_PAGES),
193 VDSO_OFFSET(VDSO_PREV_PAGES),
194 VM_READ,
195 NULL);
196
197 if (IS_ERR(vma)) {
198 ret = PTR_ERR(vma);
199 goto up_fail;
200 }
201
202 ret = remap_pfn_range(vma,
203 addr - VDSO_OFFSET(VDSO_VVAR_PAGE),
204 __pa_symbol(&__vvar_page) >> PAGE_SHIFT,
205 PAGE_SIZE,
206 PAGE_READONLY);
207
208 if (ret)
209 goto up_fail;
210
211#ifdef CONFIG_HPET_TIMER
212 if (hpet_address) {
213 ret = io_remap_pfn_range(vma,
214 addr - VDSO_OFFSET(VDSO_HPET_PAGE),
215 hpet_address >> PAGE_SHIFT,
216 PAGE_SIZE,
217 pgprot_noncached(PAGE_READONLY));
218
219 if (ret)
220 goto up_fail;
221 }
222#endif
223
Ingo Molnare6e54942006-06-27 02:53:50 -0700224 current_thread_info()->sysenter_return =
Roland McGrath6c3652e2008-01-30 13:30:42 +0100225 VDSO32_SYMBOL(addr, SYSENTER_RETURN);
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +0200226
227 up_fail:
Peter Zijlstraf7b6eb32009-06-05 14:04:51 +0200228 if (ret)
229 current->mm->context.vdso = NULL;
230
Ingo Molnare6e54942006-06-27 02:53:50 -0700231 up_write(&mm->mmap_sem);
Jeremy Fitzhardinge1dbf527c2007-05-02 19:27:12 +0200232
Ingo Molnare6e54942006-06-27 02:53:50 -0700233 return ret;
Ingo Molnare6e54942006-06-27 02:53:50 -0700234}
235
Roland McGrathaf65d642008-01-30 13:30:43 +0100236#ifdef CONFIG_X86_64
237
Jiri Slabyd7a03802010-06-16 22:30:42 +0200238subsys_initcall(sysenter_setup);
Roland McGrathaf65d642008-01-30 13:30:43 +0100239
Roland McGratha97f52e2008-01-30 13:31:55 +0100240#ifdef CONFIG_SYSCTL
241/* Register vsyscall32 into the ABI table */
242#include <linux/sysctl.h>
243
Joe Perchesf07d91e2013-06-13 19:37:33 -0700244static struct ctl_table abi_table2[] = {
Roland McGratha97f52e2008-01-30 13:31:55 +0100245 {
246 .procname = "vsyscall32",
247 .data = &sysctl_vsyscall32,
248 .maxlen = sizeof(int),
249 .mode = 0644,
250 .proc_handler = proc_dointvec
251 },
252 {}
253};
254
Joe Perchesf07d91e2013-06-13 19:37:33 -0700255static struct ctl_table abi_root_table2[] = {
Roland McGratha97f52e2008-01-30 13:31:55 +0100256 {
Roland McGratha97f52e2008-01-30 13:31:55 +0100257 .procname = "abi",
258 .mode = 0555,
259 .child = abi_table2
260 },
261 {}
262};
263
264static __init int ia32_binfmt_init(void)
265{
266 register_sysctl_table(abi_root_table2);
267 return 0;
268}
269__initcall(ia32_binfmt_init);
270#endif
271
Roland McGrathaf65d642008-01-30 13:30:43 +0100272#else /* CONFIG_X86_32 */
273
Ingo Molnare6e54942006-06-27 02:53:50 -0700274const char *arch_vma_name(struct vm_area_struct *vma)
275{
276 if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso)
277 return "[vdso]";
278 return NULL;
279}
280
Stephen Wilson31db58b2011-03-13 15:49:15 -0400281struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
Ingo Molnare6e54942006-06-27 02:53:50 -0700282{
283 return NULL;
284}
285
Stephen Wilson83b964b2011-03-13 15:49:16 -0400286int in_gate_area(struct mm_struct *mm, unsigned long addr)
Ingo Molnare6e54942006-06-27 02:53:50 -0700287{
Andy Lutomirskib0b49f22014-03-13 16:01:26 -0700288 return 0;
Ingo Molnare6e54942006-06-27 02:53:50 -0700289}
290
Stephen Wilsoncae5d392011-03-13 15:49:17 -0400291int in_gate_area_no_mm(unsigned long addr)
Ingo Molnare6e54942006-06-27 02:53:50 -0700292{
293 return 0;
294}
Roland McGrathaf65d642008-01-30 13:30:43 +0100295
296#endif /* CONFIG_X86_64 */