blob: 3a45fcd0b924c42f7a25c19fbf4ff984eabad975 [file] [log] [blame]
Ingo Molnar0c867532015-04-22 10:53:34 +02001/*
Ingo Molnarae026792015-04-26 15:36:46 +02002 * x86 FPU boot time init code:
Ingo Molnar0c867532015-04-22 10:53:34 +02003 */
Ingo Molnar78f7f1e2015-04-24 02:54:44 +02004#include <asm/fpu/internal.h>
Ingo Molnar0c867532015-04-22 10:53:34 +02005#include <asm/tlbflush.h>
yu-cheng yu4f81cba2016-01-06 14:24:51 -08006#include <asm/setup.h>
7#include <asm/cmdline.h>
Ingo Molnar0c867532015-04-22 10:53:34 +02008
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +02009#include <linux/sched.h>
yu-cheng yu4f81cba2016-01-06 14:24:51 -080010#include <linux/init.h>
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +020011
Ingo Molnarae026792015-04-26 15:36:46 +020012/*
13 * Initialize the TS bit in CR0 according to the style of context-switches
14 * we are using:
15 */
Ingo Molnar41e78412015-04-26 15:32:40 +020016static void fpu__init_cpu_ctx_switch(void)
17{
Borislav Petkov362f9242015-12-07 10:39:41 +010018 if (!boot_cpu_has(X86_FEATURE_EAGER_FPU))
Ingo Molnar41e78412015-04-26 15:32:40 +020019 stts();
20 else
21 clts();
22}
23
24/*
25 * Initialize the registers found in all CPUs, CR0 and CR4:
26 */
27static void fpu__init_cpu_generic(void)
28{
29 unsigned long cr0;
30 unsigned long cr4_mask = 0;
31
32 if (cpu_has_fxsr)
33 cr4_mask |= X86_CR4_OSFXSR;
34 if (cpu_has_xmm)
35 cr4_mask |= X86_CR4_OSXMMEXCPT;
36 if (cr4_mask)
37 cr4_set_bits(cr4_mask);
38
39 cr0 = read_cr0();
40 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
41 if (!cpu_has_fpu)
42 cr0 |= X86_CR0_EM;
43 write_cr0(cr0);
Ingo Molnarb1276c42015-04-29 10:58:03 +020044
45 /* Flush out any pending x87 state: */
Ingo Molnar5fc96032015-08-22 09:52:06 +020046#ifdef CONFIG_MATH_EMULATION
47 if (!cpu_has_fpu)
48 fpstate_init_soft(&current->thread.fpu.state.soft);
49 else
50#endif
51 asm volatile ("fninit");
Ingo Molnar41e78412015-04-26 15:32:40 +020052}
53
54/*
Ingo Molnarae026792015-04-26 15:36:46 +020055 * Enable all supported FPU features. Called when a CPU is brought online:
Ingo Molnar41e78412015-04-26 15:32:40 +020056 */
57void fpu__init_cpu(void)
58{
59 fpu__init_cpu_generic();
60 fpu__init_cpu_xstate();
61 fpu__init_cpu_ctx_switch();
62}
63
Ingo Molnar4d164092015-04-22 13:44:25 +020064/*
Ingo Molnardd863882015-04-26 15:07:18 +020065 * The earliest FPU detection code.
66 *
67 * Set the X86_FEATURE_FPU CPU-capability bit based on
68 * trying to execute an actual sequence of FPU instructions:
Ingo Molnar2e2f3da2015-04-26 14:40:54 +020069 */
70static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
71{
72 unsigned long cr0;
73 u16 fsw, fcw;
74
75 fsw = fcw = 0xffff;
76
77 cr0 = read_cr0();
78 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
79 write_cr0(cr0);
80
81 asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
82 : "+m" (fsw), "+m" (fcw));
83
84 if (fsw == 0 && (fcw & 0x103f) == 0x003f)
85 set_cpu_cap(c, X86_FEATURE_FPU);
86 else
87 clear_cpu_cap(c, X86_FEATURE_FPU);
Ingo Molnare83ab9a2015-04-26 14:43:44 +020088
89#ifndef CONFIG_MATH_EMULATION
90 if (!cpu_has_fpu) {
Ingo Molnarae026792015-04-26 15:36:46 +020091 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
Ingo Molnare83ab9a2015-04-26 14:43:44 +020092 for (;;)
93 asm volatile("hlt");
94 }
95#endif
Ingo Molnar2e2f3da2015-04-26 14:40:54 +020096}
97
98/*
Ingo Molnar4d164092015-04-22 13:44:25 +020099 * Boot time FPU feature detection code:
100 */
Ingo Molnar0c867532015-04-22 10:53:34 +0200101unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200102
Ingo Molnar32231872015-05-04 09:52:42 +0200103static void __init fpu__init_system_mxcsr(void)
Ingo Molnar0c867532015-04-22 10:53:34 +0200104{
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200105 unsigned int mask = 0;
Ingo Molnar0c867532015-04-22 10:53:34 +0200106
107 if (cpu_has_fxsr) {
Ingo Molnarb96fecb2015-07-04 09:58:19 +0200108 /* Static because GCC does not get 16-byte stack alignment right: */
109 static struct fxregs_state fxregs __initdata;
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200110
Ingo Molnarb96fecb2015-07-04 09:58:19 +0200111 asm volatile("fxsave %0" : "+m" (fxregs));
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200112
Ingo Molnarb96fecb2015-07-04 09:58:19 +0200113 mask = fxregs.mxcsr_mask;
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200114
115 /*
116 * If zero then use the default features mask,
117 * which has all features set, except the
118 * denormals-are-zero feature bit:
119 */
Ingo Molnar0c867532015-04-22 10:53:34 +0200120 if (mask == 0)
121 mask = 0x0000ffbf;
122 }
123 mxcsr_feature_mask &= mask;
124}
125
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200126/*
127 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
128 */
Ingo Molnar32231872015-05-04 09:52:42 +0200129static void __init fpu__init_system_generic(void)
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200130{
131 /*
132 * Set up the legacy init FPU context. (xstate init might overwrite this
133 * with a more modern format, if the CPU supports it.)
134 */
Ingo Molnar6f575022015-04-30 11:07:06 +0200135 fpstate_init_fxstate(&init_fpstate.fxsave);
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200136
137 fpu__init_system_mxcsr();
138}
139
Ingo Molnarae026792015-04-26 15:36:46 +0200140/*
141 * Size of the FPU context state. All tasks in the system use the
142 * same context size, regardless of what portion they use.
143 * This is inherent to the XSAVE architecture which puts all state
144 * components into a single, continuous memory block:
145 */
Ingo Molnar41e78412015-04-26 15:32:40 +0200146unsigned int xstate_size;
147EXPORT_SYMBOL_GPL(xstate_size);
148
Jiri Olsa25ec02f2015-12-21 15:25:30 +0100149/* Get alignment of the TYPE. */
150#define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
151
152/*
153 * Enforce that 'MEMBER' is the last field of 'TYPE'.
154 *
155 * Align the computed size with alignment of the TYPE,
156 * because that's how C aligns structs.
157 */
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200158#define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
Jiri Olsa25ec02f2015-12-21 15:25:30 +0100159 BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
160 TYPE_ALIGN(TYPE)))
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200161
162/*
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200163 * We append the 'struct fpu' to the task_struct:
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200164 */
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200165static void __init fpu__init_task_struct_size(void)
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200166{
167 int task_size = sizeof(struct task_struct);
168
169 /*
170 * Subtract off the static size of the register state.
171 * It potentially has a bunch of padding.
172 */
173 task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
174
175 /*
176 * Add back the dynamically-calculated register state
177 * size.
178 */
179 task_size += xstate_size;
180
181 /*
182 * We dynamically size 'struct fpu', so we require that
183 * it be at the end of 'thread_struct' and that
184 * 'thread_struct' be at the end of 'task_struct'. If
185 * you hit a compile error here, check the structure to
186 * see if something got added to the end.
187 */
188 CHECK_MEMBER_AT_END_OF(struct fpu, state);
189 CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
190 CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
191
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200192 arch_task_struct_size = task_size;
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200193}
194
Ingo Molnar41e78412015-04-26 15:32:40 +0200195/*
196 * Set up the xstate_size based on the legacy FPU context size.
197 *
198 * We set this up first, and later it will be overwritten by
199 * fpu__init_system_xstate() if the CPU knows about xstates.
200 */
Ingo Molnar32231872015-05-04 09:52:42 +0200201static void __init fpu__init_system_xstate_size_legacy(void)
Ingo Molnar0c867532015-04-22 10:53:34 +0200202{
Rasmus Villemoese49a4492015-11-13 15:18:31 +0100203 static int on_boot_cpu __initdata = 1;
Ingo Molnare97131a2015-05-05 11:34:49 +0200204
205 WARN_ON_FPU(!on_boot_cpu);
206 on_boot_cpu = 0;
207
Ingo Molnar0c867532015-04-22 10:53:34 +0200208 /*
209 * Note that xstate_size might be overwriten later during
Ingo Molnarc42103b2015-04-25 06:52:53 +0200210 * fpu__init_system_xstate().
Ingo Molnar0c867532015-04-22 10:53:34 +0200211 */
212
213 if (!cpu_has_fpu) {
214 /*
215 * Disable xsave as we do not support it if i387
216 * emulation is enabled.
217 */
218 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
219 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
Ingo Molnarc47ada32015-04-30 17:15:32 +0200220 xstate_size = sizeof(struct swregs_state);
Ingo Molnar6a133202015-04-25 04:29:26 +0200221 } else {
222 if (cpu_has_fxsr)
Ingo Molnarc47ada32015-04-30 17:15:32 +0200223 xstate_size = sizeof(struct fxregs_state);
Ingo Molnar6a133202015-04-25 04:29:26 +0200224 else
Ingo Molnarc47ada32015-04-30 17:15:32 +0200225 xstate_size = sizeof(struct fregs_state);
Ingo Molnar0c867532015-04-22 10:53:34 +0200226 }
Ingo Molnar6f56a8d2015-05-20 11:59:45 +0200227 /*
228 * Quirk: we don't yet handle the XSAVES* instructions
229 * correctly, as we don't correctly convert between
230 * standard and compacted format when interfacing
231 * with user-space - so disable it for now.
232 *
233 * The difference is small: with recent CPUs the
234 * compacted format is only marginally smaller than
235 * the standard FPU state format.
236 *
237 * ( This is easy to backport while we are fixing
238 * XSAVES* support. )
239 */
240 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
Ingo Molnar0c867532015-04-22 10:53:34 +0200241}
242
Ingo Molnarae026792015-04-26 15:36:46 +0200243/*
244 * FPU context switching strategies:
245 *
246 * Against popular belief, we don't do lazy FPU saves, due to the
247 * task migration complications it brings on SMP - we only do
248 * lazy FPU restores.
249 *
250 * 'lazy' is the traditional strategy, which is based on setting
251 * CR0::TS to 1 during context-switch (instead of doing a full
252 * restore of the FPU state), which causes the first FPU instruction
253 * after the context switch (whenever it is executed) to fault - at
254 * which point we lazily restore the FPU state into FPU registers.
255 *
256 * Tasks are of course under no obligation to execute FPU instructions,
257 * so it can easily happen that another context-switch occurs without
258 * a single FPU instruction being executed. If we eventually switch
259 * back to the original task (that still owns the FPU) then we have
260 * not only saved the restores along the way, but we also have the
261 * FPU ready to be used for the original task.
262 *
263 * 'eager' switching is used on modern CPUs, there we switch the FPU
264 * state during every context switch, regardless of whether the task
265 * has used FPU instructions in that time slice or not. This is done
266 * because modern FPU context saving instructions are able to optimize
267 * state saving and restoration in hardware: they can detect both
268 * unused and untouched FPU state and optimize accordingly.
269 *
270 * [ Note that even in 'lazy' mode we might optimize context switches
271 * to use 'eager' restores, if we detect that a task is using the FPU
272 * frequently. See the fpu->counter logic in fpu/internal.h for that. ]
273 */
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200274static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO;
275
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200276/*
Ingo Molnarae026792015-04-26 15:36:46 +0200277 * Pick the FPU context switching strategy:
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200278 */
Ingo Molnar32231872015-05-04 09:52:42 +0200279static void __init fpu__init_system_ctx_switch(void)
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200280{
Rasmus Villemoese49a4492015-11-13 15:18:31 +0100281 static bool on_boot_cpu __initdata = 1;
Ingo Molnare97131a2015-05-05 11:34:49 +0200282
283 WARN_ON_FPU(!on_boot_cpu);
284 on_boot_cpu = 0;
285
286 WARN_ON_FPU(current->thread.fpu.fpstate_active);
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200287 current_thread_info()->status = 0;
288
289 /* Auto enable eagerfpu for xsaveopt */
Borislav Petkov362f9242015-12-07 10:39:41 +0100290 if (boot_cpu_has(X86_FEATURE_XSAVEOPT) && eagerfpu != DISABLE)
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200291 eagerfpu = ENABLE;
292
Dave Hansend91cab72015-09-02 16:31:26 -0700293 if (xfeatures_mask & XFEATURE_MASK_EAGER) {
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200294 if (eagerfpu == DISABLE) {
295 pr_err("x86/fpu: eagerfpu switching disabled, disabling the following xstate features: 0x%llx.\n",
Dave Hansend91cab72015-09-02 16:31:26 -0700296 xfeatures_mask & XFEATURE_MASK_EAGER);
297 xfeatures_mask &= ~XFEATURE_MASK_EAGER;
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200298 } else {
299 eagerfpu = ENABLE;
300 }
301 }
302
303 if (eagerfpu == ENABLE)
304 setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
305
Ingo Molnar32231872015-05-04 09:52:42 +0200306 printk(KERN_INFO "x86/fpu: Using '%s' FPU context switches.\n", eagerfpu == ENABLE ? "eager" : "lazy");
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200307}
308
Ingo Molnare35f6f12015-04-25 04:34:48 +0200309/*
yu-cheng yu4f81cba2016-01-06 14:24:51 -0800310 * We parse fpu parameters early because fpu__init_system() is executed
311 * before parse_early_param().
312 */
313static void __init fpu__init_parse_early_param(void)
314{
315 /*
316 * No need to check "eagerfpu=auto" again, since it is the
317 * initial default.
318 */
319 if (cmdline_find_option_bool(boot_command_line, "eagerfpu=off"))
320 eagerfpu = DISABLE;
321 else if (cmdline_find_option_bool(boot_command_line, "eagerfpu=on"))
322 eagerfpu = ENABLE;
323
324 if (cmdline_find_option_bool(boot_command_line, "no387"))
325 setup_clear_cpu_cap(X86_FEATURE_FPU);
326
327 if (cmdline_find_option_bool(boot_command_line, "nofxsr")) {
328 setup_clear_cpu_cap(X86_FEATURE_FXSR);
329 setup_clear_cpu_cap(X86_FEATURE_FXSR_OPT);
330 setup_clear_cpu_cap(X86_FEATURE_XMM);
331 }
332
333 if (cmdline_find_option_bool(boot_command_line, "noxsave"))
334 fpu__xstate_clear_all_cpu_caps();
335
336 if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
337 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
338
339 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
340 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
341}
342
343/*
Ingo Molnarae026792015-04-26 15:36:46 +0200344 * Called on the boot CPU once per system bootup, to set up the initial
345 * FPU state that is later cloned into all processes:
Ingo Molnare35f6f12015-04-25 04:34:48 +0200346 */
Ingo Molnar32231872015-05-04 09:52:42 +0200347void __init fpu__init_system(struct cpuinfo_x86 *c)
Ingo Molnare35f6f12015-04-25 04:34:48 +0200348{
yu-cheng yu4f81cba2016-01-06 14:24:51 -0800349 fpu__init_parse_early_param();
Ingo Molnardd863882015-04-26 15:07:18 +0200350 fpu__init_system_early_generic(c);
351
Ingo Molnarae026792015-04-26 15:36:46 +0200352 /*
353 * The FPU has to be operational for some of the
354 * later FPU init activities:
355 */
Ingo Molnare35f6f12015-04-25 04:34:48 +0200356 fpu__init_cpu();
Ingo Molnar0c867532015-04-22 10:53:34 +0200357
Ingo Molnar2507e1c2015-04-25 08:35:53 +0200358 /*
Ingo Molnarae026792015-04-26 15:36:46 +0200359 * But don't leave CR0::TS set yet, as some of the FPU setup
360 * methods depend on being able to execute FPU instructions
361 * that will fault on a set TS, such as the FXSAVE in
362 * fpu__init_system_mxcsr().
Ingo Molnar530b37e2015-04-25 08:27:44 +0200363 */
364 clts();
365
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200366 fpu__init_system_generic();
Ingo Molnar7638b742015-04-26 15:23:37 +0200367 fpu__init_system_xstate_size_legacy();
Ingo Molnarc42103b2015-04-25 06:52:53 +0200368 fpu__init_system_xstate();
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200369 fpu__init_task_struct_size();
Ingo Molnar997578b2015-04-26 10:35:57 +0200370
Ingo Molnar011545b2015-04-26 08:28:31 +0200371 fpu__init_system_ctx_switch();
Ingo Molnar0c867532015-04-22 10:53:34 +0200372}