H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 1 | #ifndef _ASM_X86_PROCESSOR_H |
| 2 | #define _ASM_X86_PROCESSOR_H |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 3 | |
Glauber de Oliveira Costa | 053de04 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 4 | #include <asm/processor-flags.h> |
| 5 | |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 6 | /* Forward declaration, a strange C thing */ |
| 7 | struct task_struct; |
| 8 | struct mm_struct; |
Brian Gerst | 9fda6a0 | 2015-07-29 01:41:16 -0400 | [diff] [blame] | 9 | struct vm86; |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 10 | |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 11 | #include <asm/math_emu.h> |
| 12 | #include <asm/segment.h> |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 13 | #include <asm/types.h> |
Ingo Molnar | decb4c4 | 2015-09-05 09:32:43 +0200 | [diff] [blame] | 14 | #include <uapi/asm/sigcontext.h> |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 15 | #include <asm/current.h> |
Borislav Petkov | cd4d09e | 2016-01-26 22:12:04 +0100 | [diff] [blame] | 16 | #include <asm/cpufeatures.h> |
Glauber de Oliveira Costa | c72dcf8 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 17 | #include <asm/page.h> |
Jeremy Fitzhardinge | 54321d9 | 2009-02-11 10:20:05 -0800 | [diff] [blame] | 18 | #include <asm/pgtable_types.h> |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 19 | #include <asm/percpu.h> |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 20 | #include <asm/msr.h> |
| 21 | #include <asm/desc_defs.h> |
Andi Kleen | bd61643 | 2008-01-30 13:32:38 +0100 | [diff] [blame] | 22 | #include <asm/nops.h> |
David Howells | f05e798 | 2012-03-28 18:11:12 +0100 | [diff] [blame] | 23 | #include <asm/special_insns.h> |
Ingo Molnar | 14b9675 | 2015-04-22 09:57:24 +0200 | [diff] [blame] | 24 | #include <asm/fpu/types.h> |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 25 | |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 26 | #include <linux/personality.h> |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 27 | #include <linux/cache.h> |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 28 | #include <linux/threads.h> |
Peter Zijlstra | 5cbc19a | 2009-09-02 11:49:52 +0200 | [diff] [blame] | 29 | #include <linux/math64.h> |
Peter Zijlstra | faa4602 | 2010-03-25 14:51:50 +0100 | [diff] [blame] | 30 | #include <linux/err.h> |
David Howells | f05e798 | 2012-03-28 18:11:12 +0100 | [diff] [blame] | 31 | #include <linux/irqflags.h> |
Tom Lendacky | 21729f8 | 2017-07-17 16:10:07 -0500 | [diff] [blame] | 32 | #include <linux/mem_encrypt.h> |
David Howells | f05e798 | 2012-03-28 18:11:12 +0100 | [diff] [blame] | 33 | |
| 34 | /* |
| 35 | * We handle most unaligned accesses in hardware. On the other hand |
| 36 | * unaligned DMA can be quite expensive on some Nehalem processors. |
| 37 | * |
| 38 | * Based on this we disable the IP header alignment in network drivers. |
| 39 | */ |
| 40 | #define NET_IP_ALIGN 0 |
Glauber de Oliveira Costa | c72dcf8 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 41 | |
K.Prasad | b332828c | 2009-06-01 23:43:10 +0530 | [diff] [blame] | 42 | #define HBP_NUM 4 |
Glauber de Oliveira Costa | 0ccb8ac | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 43 | /* |
| 44 | * Default implementation of macro that returns current |
| 45 | * instruction pointer ("program counter"). |
| 46 | */ |
| 47 | static inline void *current_text_addr(void) |
| 48 | { |
| 49 | void *pc; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 50 | |
| 51 | asm volatile("mov $1f, %0; 1:":"=r" (pc)); |
| 52 | |
Glauber de Oliveira Costa | 0ccb8ac | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 53 | return pc; |
| 54 | } |
| 55 | |
Ingo Molnar | b8c1b8ea | 2015-05-24 09:58:12 +0200 | [diff] [blame] | 56 | /* |
| 57 | * These alignment constraints are for performance in the vSMP case, |
| 58 | * but in the task_struct case we must also meet hardware imposed |
| 59 | * alignment requirements of the FPU state: |
| 60 | */ |
Glauber de Oliveira Costa | dbcb466 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 61 | #ifdef CONFIG_X86_VSMP |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 62 | # define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT) |
| 63 | # define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT) |
Glauber de Oliveira Costa | dbcb466 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 64 | #else |
Ingo Molnar | b8c1b8ea | 2015-05-24 09:58:12 +0200 | [diff] [blame] | 65 | # define ARCH_MIN_TASKALIGN __alignof__(union fpregs_state) |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 66 | # define ARCH_MIN_MMSTRUCT_ALIGN 0 |
Glauber de Oliveira Costa | dbcb466 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 67 | #endif |
| 68 | |
Alex Shi | e0ba94f | 2012-06-28 09:02:16 +0800 | [diff] [blame] | 69 | enum tlb_infos { |
| 70 | ENTRIES, |
| 71 | NR_INFO |
| 72 | }; |
| 73 | |
| 74 | extern u16 __read_mostly tlb_lli_4k[NR_INFO]; |
| 75 | extern u16 __read_mostly tlb_lli_2m[NR_INFO]; |
| 76 | extern u16 __read_mostly tlb_lli_4m[NR_INFO]; |
| 77 | extern u16 __read_mostly tlb_lld_4k[NR_INFO]; |
| 78 | extern u16 __read_mostly tlb_lld_2m[NR_INFO]; |
| 79 | extern u16 __read_mostly tlb_lld_4m[NR_INFO]; |
Kirill A. Shutemov | dd36039 | 2013-12-23 14:16:58 +0200 | [diff] [blame] | 80 | extern u16 __read_mostly tlb_lld_1g[NR_INFO]; |
Alex Shi | c4211f4 | 2012-06-28 09:02:19 +0800 | [diff] [blame] | 81 | |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 82 | /* |
| 83 | * CPU type and hardware bug flags. Kept separately for each CPU. |
Mathias Krause | 0440211 | 2017-02-12 22:12:07 +0100 | [diff] [blame] | 84 | * Members of this structure are referenced in head_32.S, so think twice |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 85 | * before touching them. [mj] |
| 86 | */ |
| 87 | |
| 88 | struct cpuinfo_x86 { |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 89 | __u8 x86; /* CPU family */ |
| 90 | __u8 x86_vendor; /* CPU vendor */ |
| 91 | __u8 x86_model; |
| 92 | __u8 x86_mask; |
Mathias Krause | 6415813 | 2017-02-12 22:12:08 +0100 | [diff] [blame] | 93 | #ifdef CONFIG_X86_64 |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 94 | /* Number of 4K pages in DTLB/ITLB combined(in pages): */ |
H. Peter Anvin | b1882e6 | 2009-01-23 17:18:52 -0800 | [diff] [blame] | 95 | int x86_tlbsize; |
Jan Beulich | 13c6c53 | 2009-03-12 12:37:34 +0000 | [diff] [blame] | 96 | #endif |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 97 | __u8 x86_virt_bits; |
| 98 | __u8 x86_phys_bits; |
| 99 | /* CPUID returned core id bits: */ |
| 100 | __u8 x86_coreid_bits; |
Borislav Petkov | 79a8b9a | 2017-02-05 11:50:21 +0100 | [diff] [blame] | 101 | __u8 cu_id; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 102 | /* Max extended CPUID function supported: */ |
| 103 | __u32 extended_cpuid_level; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 104 | /* Maximum supported CPUID level, -1=no CPUID: */ |
| 105 | int cpuid_level; |
Borislav Petkov | 65fc985 | 2013-03-20 15:07:23 +0100 | [diff] [blame] | 106 | __u32 x86_capability[NCAPINTS + NBUGINTS]; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 107 | char x86_vendor_id[16]; |
| 108 | char x86_model_id[64]; |
| 109 | /* in KB - valid for CPUS which support this call: */ |
| 110 | int x86_cache_size; |
| 111 | int x86_cache_alignment; /* In bytes */ |
Peter P Waskiewicz Jr | cbc82b1 | 2015-01-23 18:45:43 +0000 | [diff] [blame] | 112 | /* Cache QoS architectural values: */ |
| 113 | int x86_cache_max_rmid; /* max index */ |
| 114 | int x86_cache_occ_scale; /* scale to bytes */ |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 115 | int x86_power; |
| 116 | unsigned long loops_per_jiffy; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 117 | /* cpuid returned max cores value: */ |
| 118 | u16 x86_max_cores; |
| 119 | u16 apicid; |
Yinghai Lu | 01aaea1 | 2008-03-06 13:46:39 -0800 | [diff] [blame] | 120 | u16 initial_apicid; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 121 | u16 x86_clflush_size; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 122 | /* number of cores as seen by the OS: */ |
| 123 | u16 booted_cores; |
| 124 | /* Physical processor id: */ |
| 125 | u16 phys_proc_id; |
Thomas Gleixner | 1f12e32 | 2016-02-22 22:19:15 +0000 | [diff] [blame] | 126 | /* Logical processor id: */ |
| 127 | u16 logical_proc_id; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 128 | /* Core id: */ |
| 129 | u16 cpu_core_id; |
| 130 | /* Index into per_cpu list: */ |
| 131 | u16 cpu_index; |
Andi Kleen | 506ed6b | 2011-10-12 17:46:33 -0700 | [diff] [blame] | 132 | u32 microcode; |
Jan Beulich | 2c773dd | 2014-11-04 08:26:42 +0000 | [diff] [blame] | 133 | }; |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 134 | |
He Chen | 47f10a3 | 2016-11-11 17:25:34 +0800 | [diff] [blame] | 135 | struct cpuid_regs { |
| 136 | u32 eax, ebx, ecx, edx; |
| 137 | }; |
| 138 | |
| 139 | enum cpuid_regs_idx { |
| 140 | CPUID_EAX = 0, |
| 141 | CPUID_EBX, |
| 142 | CPUID_ECX, |
| 143 | CPUID_EDX, |
| 144 | }; |
| 145 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 146 | #define X86_VENDOR_INTEL 0 |
| 147 | #define X86_VENDOR_CYRIX 1 |
| 148 | #define X86_VENDOR_AMD 2 |
| 149 | #define X86_VENDOR_UMC 3 |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 150 | #define X86_VENDOR_CENTAUR 5 |
| 151 | #define X86_VENDOR_TRANSMETA 7 |
| 152 | #define X86_VENDOR_NSC 8 |
| 153 | #define X86_VENDOR_NUM 9 |
| 154 | |
| 155 | #define X86_VENDOR_UNKNOWN 0xff |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 156 | |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 157 | /* |
| 158 | * capabilities of CPUs |
| 159 | */ |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 160 | extern struct cpuinfo_x86 boot_cpu_data; |
| 161 | extern struct cpuinfo_x86 new_cpu_data; |
| 162 | |
| 163 | extern struct tss_struct doublefault_tss; |
Yinghai Lu | 3e0c373 | 2009-05-09 23:47:42 -0700 | [diff] [blame] | 164 | extern __u32 cpu_caps_cleared[NCAPINTS]; |
| 165 | extern __u32 cpu_caps_set[NCAPINTS]; |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 166 | |
| 167 | #ifdef CONFIG_SMP |
Jan Beulich | 2c773dd | 2014-11-04 08:26:42 +0000 | [diff] [blame] | 168 | DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info); |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 169 | #define cpu_data(cpu) per_cpu(cpu_info, cpu) |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 170 | #else |
Tejun Heo | 7b543a5 | 2010-12-18 16:30:05 +0100 | [diff] [blame] | 171 | #define cpu_info boot_cpu_data |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 172 | #define cpu_data(cpu) boot_cpu_data |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 173 | #endif |
| 174 | |
Jaswinder Singh | 1c6c727 | 2008-07-21 22:40:37 +0530 | [diff] [blame] | 175 | extern const struct seq_operations cpuinfo_op; |
| 176 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 177 | #define cache_line_size() (boot_cpu_data.x86_cache_alignment) |
| 178 | |
| 179 | extern void cpu_detect(struct cpuinfo_x86 *c); |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 180 | |
Yinghai Lu | f580366 | 2008-06-21 03:24:19 -0700 | [diff] [blame] | 181 | extern void early_cpu_init(void); |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 182 | extern void identify_boot_cpu(void); |
| 183 | extern void identify_secondary_cpu(struct cpuinfo_x86 *); |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 184 | extern void print_cpu_info(struct cpuinfo_x86 *); |
Yinghai Lu | 21c3fcf | 2012-02-12 09:53:57 -0800 | [diff] [blame] | 185 | void print_cpu_msr(struct cpuinfo_x86 *); |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 186 | extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c); |
He Chen | 47bdf33 | 2016-11-11 17:25:35 +0800 | [diff] [blame] | 187 | extern u32 get_scattered_cpuid_leaf(unsigned int level, |
| 188 | unsigned int sub_leaf, |
| 189 | enum cpuid_regs_idx reg); |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 190 | extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c); |
Andreas Herrmann | 04a1541 | 2012-10-19 10:59:33 +0200 | [diff] [blame] | 191 | extern void init_amd_cacheinfo(struct cpuinfo_x86 *c); |
Glauber de Oliveira Costa | 5300db8 | 2008-01-30 13:31:33 +0100 | [diff] [blame] | 192 | |
Suresh Siddha | bbb65d2 | 2008-08-23 17:47:10 +0200 | [diff] [blame] | 193 | extern void detect_extended_topology(struct cpuinfo_x86 *c); |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 194 | extern void detect_ht(struct cpuinfo_x86 *c); |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 195 | |
Fenghua Yu | d288e1c | 2012-12-20 23:44:23 -0800 | [diff] [blame] | 196 | #ifdef CONFIG_X86_32 |
| 197 | extern int have_cpuid_p(void); |
| 198 | #else |
| 199 | static inline int have_cpuid_p(void) |
| 200 | { |
| 201 | return 1; |
| 202 | } |
| 203 | #endif |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 204 | static inline void native_cpuid(unsigned int *eax, unsigned int *ebx, |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 205 | unsigned int *ecx, unsigned int *edx) |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 206 | { |
| 207 | /* ecx is often an input as well as an output. */ |
Suresh Siddha | 45a94d7 | 2009-12-16 16:25:42 -0800 | [diff] [blame] | 208 | asm volatile("cpuid" |
Joe Perches | cca2e6f | 2008-03-23 01:03:15 -0700 | [diff] [blame] | 209 | : "=a" (*eax), |
| 210 | "=b" (*ebx), |
| 211 | "=c" (*ecx), |
| 212 | "=d" (*edx) |
Andi Kleen | 506ed6b | 2011-10-12 17:46:33 -0700 | [diff] [blame] | 213 | : "0" (*eax), "2" (*ecx) |
| 214 | : "memory"); |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 215 | } |
| 216 | |
Borislav Petkov | 5dedade | 2017-01-09 12:41:43 +0100 | [diff] [blame] | 217 | #define native_cpuid_reg(reg) \ |
| 218 | static inline unsigned int native_cpuid_##reg(unsigned int op) \ |
| 219 | { \ |
| 220 | unsigned int eax = op, ebx, ecx = 0, edx; \ |
| 221 | \ |
| 222 | native_cpuid(&eax, &ebx, &ecx, &edx); \ |
| 223 | \ |
| 224 | return reg; \ |
| 225 | } |
| 226 | |
| 227 | /* |
| 228 | * Native CPUID functions returning a single datum. |
| 229 | */ |
| 230 | native_cpuid_reg(eax) |
| 231 | native_cpuid_reg(ebx) |
| 232 | native_cpuid_reg(ecx) |
| 233 | native_cpuid_reg(edx) |
| 234 | |
Andy Lutomirski | 6c690ee | 2017-06-12 10:26:14 -0700 | [diff] [blame] | 235 | /* |
| 236 | * Friendlier CR3 helpers. |
| 237 | */ |
| 238 | static inline unsigned long read_cr3_pa(void) |
| 239 | { |
| 240 | return __read_cr3() & CR3_ADDR_MASK; |
| 241 | } |
| 242 | |
Tom Lendacky | eef9c4a | 2017-07-17 16:10:08 -0500 | [diff] [blame^] | 243 | static inline unsigned long native_read_cr3_pa(void) |
| 244 | { |
| 245 | return __native_read_cr3() & CR3_ADDR_MASK; |
| 246 | } |
| 247 | |
Glauber de Oliveira Costa | c72dcf8 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 248 | static inline void load_cr3(pgd_t *pgdir) |
| 249 | { |
Tom Lendacky | 21729f8 | 2017-07-17 16:10:07 -0500 | [diff] [blame] | 250 | write_cr3(__sme_pa(pgdir)); |
Glauber de Oliveira Costa | c72dcf8 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 251 | } |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 252 | |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 253 | #ifdef CONFIG_X86_32 |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 254 | /* This is the TSS defined by the hardware. */ |
| 255 | struct x86_hw_tss { |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 256 | unsigned short back_link, __blh; |
| 257 | unsigned long sp0; |
| 258 | unsigned short ss0, __ss0h; |
Andy Lutomirski | cf9328c | 2015-04-02 12:41:45 -0700 | [diff] [blame] | 259 | unsigned long sp1; |
Andy Lutomirski | 76e4c49 | 2015-03-10 11:06:00 -0700 | [diff] [blame] | 260 | |
| 261 | /* |
Andy Lutomirski | cf9328c | 2015-04-02 12:41:45 -0700 | [diff] [blame] | 262 | * We don't use ring 1, so ss1 is a convenient scratch space in |
| 263 | * the same cacheline as sp0. We use ss1 to cache the value in |
| 264 | * MSR_IA32_SYSENTER_CS. When we context switch |
| 265 | * MSR_IA32_SYSENTER_CS, we first check if the new value being |
| 266 | * written matches ss1, and, if it's not, then we wrmsr the new |
| 267 | * value and update ss1. |
Andy Lutomirski | 76e4c49 | 2015-03-10 11:06:00 -0700 | [diff] [blame] | 268 | * |
Andy Lutomirski | cf9328c | 2015-04-02 12:41:45 -0700 | [diff] [blame] | 269 | * The only reason we context switch MSR_IA32_SYSENTER_CS is |
| 270 | * that we set it to zero in vm86 tasks to avoid corrupting the |
| 271 | * stack if we were to go through the sysenter path from vm86 |
| 272 | * mode. |
Andy Lutomirski | 76e4c49 | 2015-03-10 11:06:00 -0700 | [diff] [blame] | 273 | */ |
Andy Lutomirski | 76e4c49 | 2015-03-10 11:06:00 -0700 | [diff] [blame] | 274 | unsigned short ss1; /* MSR_IA32_SYSENTER_CS */ |
| 275 | |
| 276 | unsigned short __ss1h; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 277 | unsigned long sp2; |
| 278 | unsigned short ss2, __ss2h; |
| 279 | unsigned long __cr3; |
| 280 | unsigned long ip; |
| 281 | unsigned long flags; |
| 282 | unsigned long ax; |
| 283 | unsigned long cx; |
| 284 | unsigned long dx; |
| 285 | unsigned long bx; |
| 286 | unsigned long sp; |
| 287 | unsigned long bp; |
| 288 | unsigned long si; |
| 289 | unsigned long di; |
| 290 | unsigned short es, __esh; |
| 291 | unsigned short cs, __csh; |
| 292 | unsigned short ss, __ssh; |
| 293 | unsigned short ds, __dsh; |
| 294 | unsigned short fs, __fsh; |
| 295 | unsigned short gs, __gsh; |
| 296 | unsigned short ldt, __ldth; |
| 297 | unsigned short trace; |
| 298 | unsigned short io_bitmap_base; |
| 299 | |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 300 | } __attribute__((packed)); |
| 301 | #else |
| 302 | struct x86_hw_tss { |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 303 | u32 reserved1; |
| 304 | u64 sp0; |
| 305 | u64 sp1; |
| 306 | u64 sp2; |
| 307 | u64 reserved2; |
| 308 | u64 ist[7]; |
| 309 | u32 reserved3; |
| 310 | u32 reserved4; |
| 311 | u16 reserved5; |
| 312 | u16 io_bitmap_base; |
| 313 | |
Andy Lutomirski | d3273de | 2017-02-20 08:56:13 -0800 | [diff] [blame] | 314 | } __attribute__((packed)); |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 315 | #endif |
| 316 | |
| 317 | /* |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 318 | * IO-bitmap sizes: |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 319 | */ |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 320 | #define IO_BITMAP_BITS 65536 |
| 321 | #define IO_BITMAP_BYTES (IO_BITMAP_BITS/8) |
| 322 | #define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long)) |
| 323 | #define IO_BITMAP_OFFSET offsetof(struct tss_struct, io_bitmap) |
| 324 | #define INVALID_IO_BITMAP_OFFSET 0x8000 |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 325 | |
| 326 | struct tss_struct { |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 327 | /* |
| 328 | * The hardware state: |
| 329 | */ |
| 330 | struct x86_hw_tss x86_tss; |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 331 | |
| 332 | /* |
| 333 | * The extra 1 is there because the CPU will access an |
| 334 | * additional byte beyond the end of the IO permission |
| 335 | * bitmap. The extra byte must be all 1 bits, and must |
| 336 | * be within the limit. |
| 337 | */ |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 338 | unsigned long io_bitmap[IO_BITMAP_LONGS + 1]; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 339 | |
Andy Lutomirski | 6dcc941 | 2016-03-09 19:00:31 -0800 | [diff] [blame] | 340 | #ifdef CONFIG_X86_32 |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 341 | /* |
Andy Lutomirski | 2a41aa4 | 2016-03-09 19:00:33 -0800 | [diff] [blame] | 342 | * Space for the temporary SYSENTER stack. |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 343 | */ |
Andy Lutomirski | 2a41aa4 | 2016-03-09 19:00:33 -0800 | [diff] [blame] | 344 | unsigned long SYSENTER_stack_canary; |
Denys Vlasenko | d828c71 | 2015-03-09 15:52:18 +0100 | [diff] [blame] | 345 | unsigned long SYSENTER_stack[64]; |
Andy Lutomirski | 6dcc941 | 2016-03-09 19:00:31 -0800 | [diff] [blame] | 346 | #endif |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 347 | |
Richard Kennedy | 84e65b0 | 2008-07-04 13:56:16 +0100 | [diff] [blame] | 348 | } ____cacheline_aligned; |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 349 | |
Andy Lutomirski | 24933b8 | 2015-03-05 19:19:05 -0800 | [diff] [blame] | 350 | DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss); |
Glauber de Oliveira Costa | ca241c7 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 351 | |
Andy Lutomirski | 4f53ab1 | 2017-02-20 08:56:09 -0800 | [diff] [blame] | 352 | /* |
| 353 | * sizeof(unsigned long) coming from an extra "long" at the end |
| 354 | * of the iobitmap. |
| 355 | * |
| 356 | * -1? seg base+limit should be pointing to the address of the |
| 357 | * last valid byte |
| 358 | */ |
| 359 | #define __KERNEL_TSS_LIMIT \ |
| 360 | (IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1) |
| 361 | |
Andy Lutomirski | a7fcf28 | 2015-03-06 17:50:19 -0800 | [diff] [blame] | 362 | #ifdef CONFIG_X86_32 |
| 363 | DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); |
| 364 | #endif |
| 365 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 366 | /* |
| 367 | * Save the original ist values for checking stack pointers during debugging |
| 368 | */ |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 369 | struct orig_ist { |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 370 | unsigned long ist[7]; |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 371 | }; |
| 372 | |
Glauber Costa | fe67620 | 2008-03-03 14:12:56 -0300 | [diff] [blame] | 373 | #ifdef CONFIG_X86_64 |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 374 | DECLARE_PER_CPU(struct orig_ist, orig_ist); |
Brian Gerst | 26f80bd | 2009-01-19 00:38:58 +0900 | [diff] [blame] | 375 | |
Brian Gerst | 947e76c | 2009-01-19 12:21:28 +0900 | [diff] [blame] | 376 | union irq_stack_union { |
| 377 | char irq_stack[IRQ_STACK_SIZE]; |
| 378 | /* |
| 379 | * GCC hardcodes the stack canary as %gs:40. Since the |
| 380 | * irq_stack is the object at %gs:0, we reserve the bottom |
| 381 | * 48 bytes of the irq stack for the canary. |
| 382 | */ |
| 383 | struct { |
| 384 | char gs_base[40]; |
| 385 | unsigned long stack_canary; |
| 386 | }; |
| 387 | }; |
| 388 | |
Andi Kleen | 277d5b4 | 2013-08-05 15:02:43 -0700 | [diff] [blame] | 389 | DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible; |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 390 | DECLARE_INIT_PER_CPU(irq_stack_union); |
| 391 | |
Brian Gerst | 26f80bd | 2009-01-19 00:38:58 +0900 | [diff] [blame] | 392 | DECLARE_PER_CPU(char *, irq_stack_ptr); |
Jaswinder Singh Rajput | 9766cdb | 2009-03-14 11:19:49 +0530 | [diff] [blame] | 393 | DECLARE_PER_CPU(unsigned int, irq_count); |
Jaswinder Singh Rajput | 9766cdb | 2009-03-14 11:19:49 +0530 | [diff] [blame] | 394 | extern asmlinkage void ignore_sysret(void); |
Tejun Heo | 60a5317 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 395 | #else /* X86_64 */ |
| 396 | #ifdef CONFIG_CC_STACKPROTECTOR |
Jeremy Fitzhardinge | 1ea0d14 | 2009-09-03 12:27:15 -0700 | [diff] [blame] | 397 | /* |
| 398 | * Make sure stack canary segment base is cached-aligned: |
| 399 | * "For Intel Atom processors, avoid non zero segment base address |
| 400 | * that is not aligned to cache line boundary at all cost." |
| 401 | * (Optim Ref Manual Assembly/Compiler Coding Rule 15.) |
| 402 | */ |
| 403 | struct stack_canary { |
| 404 | char __pad[20]; /* canary at %gs:20 */ |
| 405 | unsigned long canary; |
| 406 | }; |
Jeremy Fitzhardinge | 53f8245 | 2009-09-03 14:31:44 -0700 | [diff] [blame] | 407 | DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 408 | #endif |
Steven Rostedt | 198d208 | 2014-02-06 09:41:31 -0500 | [diff] [blame] | 409 | /* |
| 410 | * per-CPU IRQ handling stacks |
| 411 | */ |
| 412 | struct irq_stack { |
| 413 | u32 stack[THREAD_SIZE/sizeof(u32)]; |
| 414 | } __aligned(THREAD_SIZE); |
| 415 | |
| 416 | DECLARE_PER_CPU(struct irq_stack *, hardirq_stack); |
| 417 | DECLARE_PER_CPU(struct irq_stack *, softirq_stack); |
Tejun Heo | 60a5317 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 418 | #endif /* X86_64 */ |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 419 | |
Fenghua Yu | bf15a8c | 2016-05-20 10:47:06 -0700 | [diff] [blame] | 420 | extern unsigned int fpu_kernel_xstate_size; |
Fenghua Yu | a1141e0 | 2016-05-20 10:47:05 -0700 | [diff] [blame] | 421 | extern unsigned int fpu_user_xstate_size; |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 422 | |
Frederic Weisbecker | 24f1e32c | 2009-09-09 19:22:48 +0200 | [diff] [blame] | 423 | struct perf_event; |
| 424 | |
Andy Lutomirski | 13d4ea0 | 2016-07-14 13:22:57 -0700 | [diff] [blame] | 425 | typedef struct { |
| 426 | unsigned long seg; |
| 427 | } mm_segment_t; |
| 428 | |
Glauber de Oliveira Costa | cb38d37 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 429 | struct thread_struct { |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 430 | /* Cached TLS descriptors: */ |
| 431 | struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES]; |
| 432 | unsigned long sp0; |
| 433 | unsigned long sp; |
Glauber de Oliveira Costa | cb38d37 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 434 | #ifdef CONFIG_X86_32 |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 435 | unsigned long sysenter_cs; |
Glauber de Oliveira Costa | cb38d37 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 436 | #else |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 437 | unsigned short es; |
| 438 | unsigned short ds; |
| 439 | unsigned short fsindex; |
| 440 | unsigned short gsindex; |
Glauber de Oliveira Costa | cb38d37 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 441 | #endif |
Andy Lutomirski | b9d989c | 2016-09-13 14:29:21 -0700 | [diff] [blame] | 442 | |
| 443 | u32 status; /* thread synchronous flags */ |
| 444 | |
Alexey Dobriyan | d756f4ad | 2009-05-04 03:29:52 +0400 | [diff] [blame] | 445 | #ifdef CONFIG_X86_64 |
Andy Lutomirski | 296f781 | 2016-04-26 12:23:29 -0700 | [diff] [blame] | 446 | unsigned long fsbase; |
| 447 | unsigned long gsbase; |
| 448 | #else |
| 449 | /* |
| 450 | * XXX: this could presumably be unsigned short. Alternatively, |
| 451 | * 32-bit kernels could be taught to use fsindex instead. |
| 452 | */ |
| 453 | unsigned long fs; |
| 454 | unsigned long gs; |
Alexey Dobriyan | d756f4ad | 2009-05-04 03:29:52 +0400 | [diff] [blame] | 455 | #endif |
Ingo Molnar | c5bedc6 | 2015-04-23 12:49:20 +0200 | [diff] [blame] | 456 | |
Frederic Weisbecker | 24f1e32c | 2009-09-09 19:22:48 +0200 | [diff] [blame] | 457 | /* Save middle states of ptrace breakpoints */ |
| 458 | struct perf_event *ptrace_bps[HBP_NUM]; |
| 459 | /* Debug status used for traps, single steps, etc... */ |
| 460 | unsigned long debugreg6; |
Frederic Weisbecker | 326264a | 2010-02-18 18:24:18 +0100 | [diff] [blame] | 461 | /* Keep track of the exact dr7 value set by the user */ |
| 462 | unsigned long ptrace_dr7; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 463 | /* Fault info: */ |
| 464 | unsigned long cr2; |
Srikar Dronamraju | 51e7dc7 | 2012-03-12 14:55:55 +0530 | [diff] [blame] | 465 | unsigned long trap_nr; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 466 | unsigned long error_code; |
Brian Gerst | 9fda6a0 | 2015-07-29 01:41:16 -0400 | [diff] [blame] | 467 | #ifdef CONFIG_VM86 |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 468 | /* Virtual 86 mode info */ |
Brian Gerst | 9fda6a0 | 2015-07-29 01:41:16 -0400 | [diff] [blame] | 469 | struct vm86 *vm86; |
Glauber de Oliveira Costa | cb38d37 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 470 | #endif |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 471 | /* IO permissions: */ |
| 472 | unsigned long *io_bitmap_ptr; |
| 473 | unsigned long iopl; |
| 474 | /* Max allowed port in the bitmap, in bytes: */ |
| 475 | unsigned io_bitmap_max; |
Dave Hansen | 0c8c0f0 | 2015-07-17 12:28:11 +0200 | [diff] [blame] | 476 | |
Andy Lutomirski | 13d4ea0 | 2016-07-14 13:22:57 -0700 | [diff] [blame] | 477 | mm_segment_t addr_limit; |
| 478 | |
Ingo Molnar | 2a53ccb | 2016-07-15 10:21:11 +0200 | [diff] [blame] | 479 | unsigned int sig_on_uaccess_err:1; |
Andy Lutomirski | dfa9a94 | 2016-07-14 13:22:56 -0700 | [diff] [blame] | 480 | unsigned int uaccess_err:1; /* uaccess failed */ |
| 481 | |
Dave Hansen | 0c8c0f0 | 2015-07-17 12:28:11 +0200 | [diff] [blame] | 482 | /* Floating point and extended processor state */ |
| 483 | struct fpu fpu; |
| 484 | /* |
| 485 | * WARNING: 'fpu' is dynamically-sized. It *MUST* be at |
| 486 | * the end. |
| 487 | */ |
Glauber de Oliveira Costa | cb38d37 | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 488 | }; |
| 489 | |
Glauber de Oliveira Costa | 62d7d7e | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 490 | /* |
Andy Lutomirski | b9d989c | 2016-09-13 14:29:21 -0700 | [diff] [blame] | 491 | * Thread-synchronous status. |
| 492 | * |
| 493 | * This is different from the flags in that nobody else |
| 494 | * ever touches our thread-synchronous status, so we don't |
| 495 | * have to worry about atomic accesses. |
| 496 | */ |
| 497 | #define TS_COMPAT 0x0002 /* 32bit syscall active (64BIT)*/ |
| 498 | |
| 499 | /* |
Glauber de Oliveira Costa | 62d7d7e | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 500 | * Set IOPL bits in EFLAGS from given mask |
| 501 | */ |
| 502 | static inline void native_set_iopl_mask(unsigned mask) |
| 503 | { |
| 504 | #ifdef CONFIG_X86_32 |
| 505 | unsigned int reg; |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 506 | |
Joe Perches | cca2e6f | 2008-03-23 01:03:15 -0700 | [diff] [blame] | 507 | asm volatile ("pushfl;" |
| 508 | "popl %0;" |
| 509 | "andl %1, %0;" |
| 510 | "orl %2, %0;" |
| 511 | "pushl %0;" |
| 512 | "popfl" |
| 513 | : "=&r" (reg) |
| 514 | : "i" (~X86_EFLAGS_IOPL), "r" (mask)); |
Glauber de Oliveira Costa | 62d7d7e | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 515 | #endif |
| 516 | } |
| 517 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 518 | static inline void |
| 519 | native_load_sp0(struct tss_struct *tss, struct thread_struct *thread) |
Glauber de Oliveira Costa | 7818a1e | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 520 | { |
| 521 | tss->x86_tss.sp0 = thread->sp0; |
| 522 | #ifdef CONFIG_X86_32 |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 523 | /* Only happens when SEP is enabled, no need to test "SEP"arately: */ |
Glauber de Oliveira Costa | 7818a1e | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 524 | if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) { |
| 525 | tss->x86_tss.ss1 = thread->sysenter_cs; |
| 526 | wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0); |
| 527 | } |
| 528 | #endif |
| 529 | } |
Glauber de Oliveira Costa | 1b46cbe | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 530 | |
Glauber de Oliveira Costa | e801f86 | 2008-01-30 13:32:08 +0100 | [diff] [blame] | 531 | static inline void native_swapgs(void) |
| 532 | { |
| 533 | #ifdef CONFIG_X86_64 |
| 534 | asm volatile("swapgs" ::: "memory"); |
| 535 | #endif |
| 536 | } |
| 537 | |
Andy Lutomirski | a7fcf28 | 2015-03-06 17:50:19 -0800 | [diff] [blame] | 538 | static inline unsigned long current_top_of_stack(void) |
Andy Lutomirski | 8ef46a6 | 2015-03-05 19:19:02 -0800 | [diff] [blame] | 539 | { |
Andy Lutomirski | a7fcf28 | 2015-03-06 17:50:19 -0800 | [diff] [blame] | 540 | #ifdef CONFIG_X86_64 |
Andy Lutomirski | 24933b8 | 2015-03-05 19:19:05 -0800 | [diff] [blame] | 541 | return this_cpu_read_stable(cpu_tss.x86_tss.sp0); |
Andy Lutomirski | a7fcf28 | 2015-03-06 17:50:19 -0800 | [diff] [blame] | 542 | #else |
| 543 | /* sp0 on x86_32 is special in and around vm86 mode. */ |
| 544 | return this_cpu_read_stable(cpu_current_top_of_stack); |
| 545 | #endif |
Andy Lutomirski | 8ef46a6 | 2015-03-05 19:19:02 -0800 | [diff] [blame] | 546 | } |
| 547 | |
Glauber de Oliveira Costa | 7818a1e | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 548 | #ifdef CONFIG_PARAVIRT |
| 549 | #include <asm/paravirt.h> |
| 550 | #else |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 551 | #define __cpuid native_cpuid |
Glauber de Oliveira Costa | 1b46cbe | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 552 | |
Joe Perches | cca2e6f | 2008-03-23 01:03:15 -0700 | [diff] [blame] | 553 | static inline void load_sp0(struct tss_struct *tss, |
| 554 | struct thread_struct *thread) |
Glauber de Oliveira Costa | 7818a1e | 2008-01-30 13:31:31 +0100 | [diff] [blame] | 555 | { |
| 556 | native_load_sp0(tss, thread); |
| 557 | } |
| 558 | |
Glauber de Oliveira Costa | 62d7d7e | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 559 | #define set_iopl_mask native_set_iopl_mask |
Glauber de Oliveira Costa | 1b46cbe | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 560 | #endif /* CONFIG_PARAVIRT */ |
| 561 | |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 562 | /* Free all resources held by a thread. */ |
| 563 | extern void release_thread(struct task_struct *); |
| 564 | |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 565 | unsigned long get_wchan(struct task_struct *p); |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 566 | |
| 567 | /* |
| 568 | * Generic CPUID function |
| 569 | * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx |
| 570 | * resulting in stale register contents being returned. |
| 571 | */ |
| 572 | static inline void cpuid(unsigned int op, |
| 573 | unsigned int *eax, unsigned int *ebx, |
| 574 | unsigned int *ecx, unsigned int *edx) |
| 575 | { |
| 576 | *eax = op; |
| 577 | *ecx = 0; |
| 578 | __cpuid(eax, ebx, ecx, edx); |
| 579 | } |
| 580 | |
| 581 | /* Some CPUID calls want 'count' to be placed in ecx */ |
| 582 | static inline void cpuid_count(unsigned int op, int count, |
| 583 | unsigned int *eax, unsigned int *ebx, |
| 584 | unsigned int *ecx, unsigned int *edx) |
| 585 | { |
| 586 | *eax = op; |
| 587 | *ecx = count; |
| 588 | __cpuid(eax, ebx, ecx, edx); |
| 589 | } |
| 590 | |
| 591 | /* |
| 592 | * CPUID functions returning a single datum |
| 593 | */ |
| 594 | static inline unsigned int cpuid_eax(unsigned int op) |
| 595 | { |
| 596 | unsigned int eax, ebx, ecx, edx; |
| 597 | |
| 598 | cpuid(op, &eax, &ebx, &ecx, &edx); |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 599 | |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 600 | return eax; |
| 601 | } |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 602 | |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 603 | static inline unsigned int cpuid_ebx(unsigned int op) |
| 604 | { |
| 605 | unsigned int eax, ebx, ecx, edx; |
| 606 | |
| 607 | cpuid(op, &eax, &ebx, &ecx, &edx); |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 608 | |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 609 | return ebx; |
| 610 | } |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 611 | |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 612 | static inline unsigned int cpuid_ecx(unsigned int op) |
| 613 | { |
| 614 | unsigned int eax, ebx, ecx, edx; |
| 615 | |
| 616 | cpuid(op, &eax, &ebx, &ecx, &edx); |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 617 | |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 618 | return ecx; |
| 619 | } |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 620 | |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 621 | static inline unsigned int cpuid_edx(unsigned int op) |
| 622 | { |
| 623 | unsigned int eax, ebx, ecx, edx; |
| 624 | |
| 625 | cpuid(op, &eax, &ebx, &ecx, &edx); |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 626 | |
Glauber de Oliveira Costa | c758ecf | 2008-01-30 13:31:03 +0100 | [diff] [blame] | 627 | return edx; |
| 628 | } |
| 629 | |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 630 | /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ |
Denys Vlasenko | 0b101e6 | 2015-09-24 14:02:29 +0200 | [diff] [blame] | 631 | static __always_inline void rep_nop(void) |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 632 | { |
Joe Perches | cca2e6f | 2008-03-23 01:03:15 -0700 | [diff] [blame] | 633 | asm volatile("rep; nop" ::: "memory"); |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 634 | } |
| 635 | |
Denys Vlasenko | 0b101e6 | 2015-09-24 14:02:29 +0200 | [diff] [blame] | 636 | static __always_inline void cpu_relax(void) |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 637 | { |
| 638 | rep_nop(); |
| 639 | } |
| 640 | |
Andy Lutomirski | c198b12 | 2016-12-09 10:24:08 -0800 | [diff] [blame] | 641 | /* |
| 642 | * This function forces the icache and prefetched instruction stream to |
| 643 | * catch up with reality in two very specific cases: |
| 644 | * |
| 645 | * a) Text was modified using one virtual address and is about to be executed |
| 646 | * from the same physical page at a different virtual address. |
| 647 | * |
| 648 | * b) Text was modified on a different CPU, may subsequently be |
| 649 | * executed on this CPU, and you want to make sure the new version |
| 650 | * gets executed. This generally means you're calling this in a IPI. |
| 651 | * |
| 652 | * If you're calling this for a different reason, you're probably doing |
| 653 | * it wrong. |
| 654 | */ |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 655 | static inline void sync_core(void) |
| 656 | { |
Andy Lutomirski | c198b12 | 2016-12-09 10:24:08 -0800 | [diff] [blame] | 657 | /* |
| 658 | * There are quite a few ways to do this. IRET-to-self is nice |
| 659 | * because it works on every CPU, at any CPL (so it's compatible |
| 660 | * with paravirtualization), and it never exits to a hypervisor. |
| 661 | * The only down sides are that it's a bit slow (it seems to be |
| 662 | * a bit more than 2x slower than the fastest options) and that |
| 663 | * it unmasks NMIs. The "push %cs" is needed because, in |
| 664 | * paravirtual environments, __KERNEL_CS may not be a valid CS |
| 665 | * value when we do IRET directly. |
| 666 | * |
| 667 | * In case NMI unmasking or performance ever becomes a problem, |
| 668 | * the next best option appears to be MOV-to-CR2 and an |
| 669 | * unconditional jump. That sequence also works on all CPUs, |
| 670 | * but it will fault at CPL3 (i.e. Xen PV and lguest). |
| 671 | * |
| 672 | * CPUID is the conventional way, but it's nasty: it doesn't |
| 673 | * exist on some 486-like CPUs, and it usually exits to a |
| 674 | * hypervisor. |
| 675 | * |
| 676 | * Like all of Linux's memory ordering operations, this is a |
| 677 | * compiler barrier as well. |
| 678 | */ |
| 679 | register void *__sp asm(_ASM_SP); |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 680 | |
Andy Lutomirski | 1c52d85 | 2016-12-09 10:24:05 -0800 | [diff] [blame] | 681 | #ifdef CONFIG_X86_32 |
Andy Lutomirski | c198b12 | 2016-12-09 10:24:08 -0800 | [diff] [blame] | 682 | asm volatile ( |
| 683 | "pushfl\n\t" |
| 684 | "pushl %%cs\n\t" |
| 685 | "pushl $1f\n\t" |
| 686 | "iret\n\t" |
| 687 | "1:" |
| 688 | : "+r" (__sp) : : "memory"); |
H. Peter Anvin | 45c39fb | 2012-11-28 11:50:30 -0800 | [diff] [blame] | 689 | #else |
Andy Lutomirski | c198b12 | 2016-12-09 10:24:08 -0800 | [diff] [blame] | 690 | unsigned int tmp; |
| 691 | |
| 692 | asm volatile ( |
| 693 | "mov %%ss, %0\n\t" |
| 694 | "pushq %q0\n\t" |
| 695 | "pushq %%rsp\n\t" |
| 696 | "addq $8, (%%rsp)\n\t" |
| 697 | "pushfq\n\t" |
| 698 | "mov %%cs, %0\n\t" |
| 699 | "pushq %q0\n\t" |
| 700 | "pushq $1f\n\t" |
| 701 | "iretq\n\t" |
| 702 | "1:" |
| 703 | : "=&r" (tmp), "+r" (__sp) : : "cc", "memory"); |
Ben Hutchings | 5367b68 | 2009-09-10 02:53:50 +0100 | [diff] [blame] | 704 | #endif |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 705 | } |
| 706 | |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 707 | extern void select_idle_routine(const struct cpuinfo_x86 *c); |
Borislav Petkov | 07c94a3 | 2016-12-09 19:29:11 +0100 | [diff] [blame] | 708 | extern void amd_e400_c1e_apic_setup(void); |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 709 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 710 | extern unsigned long boot_option_idle_override; |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 711 | |
Thomas Renninger | d189604 | 2010-11-03 17:06:14 +0100 | [diff] [blame] | 712 | enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT, |
Len Brown | 69fb367 | 2013-02-10 01:38:39 -0500 | [diff] [blame] | 713 | IDLE_POLL}; |
Thomas Renninger | d189604 | 2010-11-03 17:06:14 +0100 | [diff] [blame] | 714 | |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 715 | extern void enable_sep_cpu(void); |
| 716 | extern int sysenter_setup(void); |
| 717 | |
Jan Kiszka | 29c8439 | 2010-05-20 21:04:29 -0500 | [diff] [blame] | 718 | extern void early_trap_init(void); |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 719 | void early_trap_pf_init(void); |
Jan Kiszka | 29c8439 | 2010-05-20 21:04:29 -0500 | [diff] [blame] | 720 | |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 721 | /* Defined in head.S */ |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 722 | extern struct desc_ptr early_gdt_descr; |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 723 | |
| 724 | extern void cpu_set_gdt(int); |
Brian Gerst | 552be87 | 2009-01-30 17:47:53 +0900 | [diff] [blame] | 725 | extern void switch_to_new_gdt(int); |
Thomas Garnier | 45fc875 | 2017-03-14 10:05:08 -0700 | [diff] [blame] | 726 | extern void load_direct_gdt(int); |
Thomas Garnier | 69218e4 | 2017-03-14 10:05:07 -0700 | [diff] [blame] | 727 | extern void load_fixmap_gdt(int); |
Jeremy Fitzhardinge | 11e3a84 | 2009-01-30 17:47:54 +0900 | [diff] [blame] | 728 | extern void load_percpu_segment(int); |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 729 | extern void cpu_init(void); |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 730 | |
Markus Metzger | c272477 | 2008-12-11 13:49:59 +0100 | [diff] [blame] | 731 | static inline unsigned long get_debugctlmsr(void) |
| 732 | { |
Peter Zijlstra | ea8e61b | 2010-03-25 14:51:51 +0100 | [diff] [blame] | 733 | unsigned long debugctlmsr = 0; |
Markus Metzger | c272477 | 2008-12-11 13:49:59 +0100 | [diff] [blame] | 734 | |
| 735 | #ifndef CONFIG_X86_DEBUGCTLMSR |
| 736 | if (boot_cpu_data.x86 < 6) |
| 737 | return 0; |
| 738 | #endif |
| 739 | rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); |
| 740 | |
Peter Zijlstra | ea8e61b | 2010-03-25 14:51:51 +0100 | [diff] [blame] | 741 | return debugctlmsr; |
Markus Metzger | c272477 | 2008-12-11 13:49:59 +0100 | [diff] [blame] | 742 | } |
| 743 | |
Jan Beulich | 5b0e508 | 2008-03-10 13:11:17 +0000 | [diff] [blame] | 744 | static inline void update_debugctlmsr(unsigned long debugctlmsr) |
| 745 | { |
| 746 | #ifndef CONFIG_X86_DEBUGCTLMSR |
| 747 | if (boot_cpu_data.x86 < 6) |
| 748 | return; |
| 749 | #endif |
| 750 | wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); |
| 751 | } |
| 752 | |
Oleg Nesterov | 9bd1190 | 2012-09-03 15:24:17 +0200 | [diff] [blame] | 753 | extern void set_task_blockstep(struct task_struct *task, bool on); |
| 754 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 755 | /* Boot loader type from the setup header: */ |
| 756 | extern int bootloader_type; |
H. Peter Anvin | 5031296 | 2009-05-07 16:54:11 -0700 | [diff] [blame] | 757 | extern int bootloader_version; |
Glauber de Oliveira Costa | 1a53905 | 2008-01-30 13:31:39 +0100 | [diff] [blame] | 758 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 759 | extern char ignore_fpu_irq; |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 760 | |
| 761 | #define HAVE_ARCH_PICK_MMAP_LAYOUT 1 |
| 762 | #define ARCH_HAS_PREFETCHW |
| 763 | #define ARCH_HAS_SPINLOCK_PREFETCH |
| 764 | |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 765 | #ifdef CONFIG_X86_32 |
Borislav Petkov | a930dc4 | 2015-01-18 17:48:18 +0100 | [diff] [blame] | 766 | # define BASE_PREFETCH "" |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 767 | # define ARCH_HAS_PREFETCH |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 768 | #else |
Borislav Petkov | a930dc4 | 2015-01-18 17:48:18 +0100 | [diff] [blame] | 769 | # define BASE_PREFETCH "prefetcht0 %P1" |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 770 | #endif |
| 771 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 772 | /* |
| 773 | * Prefetch instructions for Pentium III (+) and AMD Athlon (+) |
| 774 | * |
| 775 | * It's not worth to care about 3dnow prefetches for the K6 |
| 776 | * because they are microcoded there and very slow. |
| 777 | */ |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 778 | static inline void prefetch(const void *x) |
| 779 | { |
Borislav Petkov | a930dc4 | 2015-01-18 17:48:18 +0100 | [diff] [blame] | 780 | alternative_input(BASE_PREFETCH, "prefetchnta %P1", |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 781 | X86_FEATURE_XMM, |
Borislav Petkov | a930dc4 | 2015-01-18 17:48:18 +0100 | [diff] [blame] | 782 | "m" (*(const char *)x)); |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 783 | } |
| 784 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 785 | /* |
| 786 | * 3dnow prefetch to get an exclusive cache line. |
| 787 | * Useful for spinlocks to avoid one state transition in the |
| 788 | * cache coherency protocol: |
| 789 | */ |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 790 | static inline void prefetchw(const void *x) |
| 791 | { |
Borislav Petkov | a930dc4 | 2015-01-18 17:48:18 +0100 | [diff] [blame] | 792 | alternative_input(BASE_PREFETCH, "prefetchw %P1", |
| 793 | X86_FEATURE_3DNOWPREFETCH, |
| 794 | "m" (*(const char *)x)); |
Glauber de Oliveira Costa | ae2e15e | 2008-01-30 13:31:40 +0100 | [diff] [blame] | 795 | } |
| 796 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 797 | static inline void spin_lock_prefetch(const void *x) |
| 798 | { |
| 799 | prefetchw(x); |
| 800 | } |
| 801 | |
Andy Lutomirski | d9e05cc | 2015-03-10 11:05:59 -0700 | [diff] [blame] | 802 | #define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \ |
| 803 | TOP_OF_KERNEL_STACK_PADDING) |
| 804 | |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 805 | #ifdef CONFIG_X86_32 |
| 806 | /* |
| 807 | * User space process size: 3GB (default). |
| 808 | */ |
Dmitry Safonov | 8f3e474 | 2017-03-06 17:17:18 +0300 | [diff] [blame] | 809 | #define IA32_PAGE_OFFSET PAGE_OFFSET |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 810 | #define TASK_SIZE PAGE_OFFSET |
Ingo Molnar | d951734 | 2009-02-20 23:32:28 +0100 | [diff] [blame] | 811 | #define TASK_SIZE_MAX TASK_SIZE |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 812 | #define STACK_TOP TASK_SIZE |
| 813 | #define STACK_TOP_MAX STACK_TOP |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 814 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 815 | #define INIT_THREAD { \ |
Andy Lutomirski | d9e05cc | 2015-03-10 11:05:59 -0700 | [diff] [blame] | 816 | .sp0 = TOP_OF_INIT_STACK, \ |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 817 | .sysenter_cs = __KERNEL_CS, \ |
| 818 | .io_bitmap_ptr = NULL, \ |
Andy Lutomirski | 13d4ea0 | 2016-07-14 13:22:57 -0700 | [diff] [blame] | 819 | .addr_limit = KERNEL_DS, \ |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 820 | } |
| 821 | |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 822 | /* |
Denys Vlasenko | 5c39403 | 2015-03-13 15:09:03 +0100 | [diff] [blame] | 823 | * TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack. |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 824 | * This is necessary to guarantee that the entire "struct pt_regs" |
Uwe Kleine-König | b595076 | 2010-11-01 15:38:34 -0400 | [diff] [blame] | 825 | * is accessible even if the CPU haven't stored the SS/ESP registers |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 826 | * on the stack (interrupt gate does not save these registers |
| 827 | * when switching to the same priv ring). |
| 828 | * Therefore beware: accessing the ss/esp fields of the |
| 829 | * "struct pt_regs" is possible, but they may contain the |
| 830 | * completely wrong values. |
| 831 | */ |
Denys Vlasenko | 5c39403 | 2015-03-13 15:09:03 +0100 | [diff] [blame] | 832 | #define task_pt_regs(task) \ |
| 833 | ({ \ |
| 834 | unsigned long __ptr = (unsigned long)task_stack_page(task); \ |
| 835 | __ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; \ |
| 836 | ((struct pt_regs *)__ptr) - 1; \ |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 837 | }) |
| 838 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 839 | #define KSTK_ESP(task) (task_pt_regs(task)->sp) |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 840 | |
| 841 | #else |
| 842 | /* |
Andy Lutomirski | 07114f0 | 2014-11-04 15:46:21 -0800 | [diff] [blame] | 843 | * User space process size. 47bits minus one guard page. The guard |
| 844 | * page is necessary on Intel CPUs: if a SYSCALL instruction is at |
| 845 | * the highest possible canonical userspace address, then that |
| 846 | * syscall will enter the kernel with a non-canonical return |
| 847 | * address, and SYSRET will explode dangerously. We avoid this |
| 848 | * particular problem by preventing anything from being mapped |
| 849 | * at the maximum canonical address. |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 850 | */ |
Ingo Molnar | d951734 | 2009-02-20 23:32:28 +0100 | [diff] [blame] | 851 | #define TASK_SIZE_MAX ((1UL << 47) - PAGE_SIZE) |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 852 | |
| 853 | /* This decides where the kernel will search for a free chunk of vm |
| 854 | * space during mmap's. |
| 855 | */ |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 856 | #define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \ |
| 857 | 0xc0000000 : 0xFFFFe000) |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 858 | |
H. Peter Anvin | 6bd3300 | 2012-02-06 13:03:09 -0800 | [diff] [blame] | 859 | #define TASK_SIZE (test_thread_flag(TIF_ADDR32) ? \ |
Ingo Molnar | d951734 | 2009-02-20 23:32:28 +0100 | [diff] [blame] | 860 | IA32_PAGE_OFFSET : TASK_SIZE_MAX) |
H. Peter Anvin | 6bd3300 | 2012-02-06 13:03:09 -0800 | [diff] [blame] | 861 | #define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_ADDR32)) ? \ |
Ingo Molnar | d951734 | 2009-02-20 23:32:28 +0100 | [diff] [blame] | 862 | IA32_PAGE_OFFSET : TASK_SIZE_MAX) |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 863 | |
David Howells | 922a70d | 2008-02-08 04:19:26 -0800 | [diff] [blame] | 864 | #define STACK_TOP TASK_SIZE |
Ingo Molnar | d951734 | 2009-02-20 23:32:28 +0100 | [diff] [blame] | 865 | #define STACK_TOP_MAX TASK_SIZE_MAX |
David Howells | 922a70d | 2008-02-08 04:19:26 -0800 | [diff] [blame] | 866 | |
Andy Lutomirski | 13d4ea0 | 2016-07-14 13:22:57 -0700 | [diff] [blame] | 867 | #define INIT_THREAD { \ |
| 868 | .sp0 = TOP_OF_INIT_STACK, \ |
| 869 | .addr_limit = KERNEL_DS, \ |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 870 | } |
| 871 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 872 | #define task_pt_regs(tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1) |
Stefani Seibold | 89240ba | 2009-11-03 10:22:40 +0100 | [diff] [blame] | 873 | extern unsigned long KSTK_ESP(struct task_struct *task); |
H. J. Lu | d046ff8 | 2012-02-14 13:49:48 -0800 | [diff] [blame] | 874 | |
Glauber de Oliveira Costa | 2f66dcc | 2008-01-30 13:31:57 +0100 | [diff] [blame] | 875 | #endif /* CONFIG_X86_64 */ |
| 876 | |
Ingo Molnar | 513ad84 | 2008-02-21 05:18:40 +0100 | [diff] [blame] | 877 | extern void start_thread(struct pt_regs *regs, unsigned long new_ip, |
| 878 | unsigned long new_sp); |
| 879 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 880 | /* |
| 881 | * This decides where the kernel will search for a free chunk of vm |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 882 | * space during mmap's. |
| 883 | */ |
Dmitry Safonov | 8f3e474 | 2017-03-06 17:17:18 +0300 | [diff] [blame] | 884 | #define __TASK_UNMAPPED_BASE(task_size) (PAGE_ALIGN(task_size / 3)) |
| 885 | #define TASK_UNMAPPED_BASE __TASK_UNMAPPED_BASE(TASK_SIZE) |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 886 | |
Ingo Molnar | 4d46a89 | 2008-02-21 04:24:40 +0100 | [diff] [blame] | 887 | #define KSTK_EIP(task) (task_pt_regs(task)->ip) |
Glauber de Oliveira Costa | 683e025 | 2008-01-30 13:31:27 +0100 | [diff] [blame] | 888 | |
Erik Bosman | 529e25f | 2008-04-14 00:24:18 +0200 | [diff] [blame] | 889 | /* Get/set a process' ability to use the timestamp counter instruction */ |
| 890 | #define GET_TSC_CTL(adr) get_tsc_mode((adr)) |
| 891 | #define SET_TSC_CTL(val) set_tsc_mode((val)) |
| 892 | |
| 893 | extern int get_tsc_mode(unsigned long adr); |
| 894 | extern int set_tsc_mode(unsigned int val); |
| 895 | |
Kyle Huey | e9ea1e7 | 2017-03-20 01:16:26 -0700 | [diff] [blame] | 896 | DECLARE_PER_CPU(u64, msr_misc_features_shadow); |
| 897 | |
Dave Hansen | fe3d197 | 2014-11-14 07:18:29 -0800 | [diff] [blame] | 898 | /* Register/unregister a process' MPX related resource */ |
Dave Hansen | 46a6e0c | 2015-06-07 11:37:02 -0700 | [diff] [blame] | 899 | #define MPX_ENABLE_MANAGEMENT() mpx_enable_management() |
| 900 | #define MPX_DISABLE_MANAGEMENT() mpx_disable_management() |
Dave Hansen | fe3d197 | 2014-11-14 07:18:29 -0800 | [diff] [blame] | 901 | |
| 902 | #ifdef CONFIG_X86_INTEL_MPX |
Dave Hansen | 46a6e0c | 2015-06-07 11:37:02 -0700 | [diff] [blame] | 903 | extern int mpx_enable_management(void); |
| 904 | extern int mpx_disable_management(void); |
Dave Hansen | fe3d197 | 2014-11-14 07:18:29 -0800 | [diff] [blame] | 905 | #else |
Dave Hansen | 46a6e0c | 2015-06-07 11:37:02 -0700 | [diff] [blame] | 906 | static inline int mpx_enable_management(void) |
Dave Hansen | fe3d197 | 2014-11-14 07:18:29 -0800 | [diff] [blame] | 907 | { |
| 908 | return -EINVAL; |
| 909 | } |
Dave Hansen | 46a6e0c | 2015-06-07 11:37:02 -0700 | [diff] [blame] | 910 | static inline int mpx_disable_management(void) |
Dave Hansen | fe3d197 | 2014-11-14 07:18:29 -0800 | [diff] [blame] | 911 | { |
| 912 | return -EINVAL; |
| 913 | } |
| 914 | #endif /* CONFIG_X86_INTEL_MPX */ |
| 915 | |
Borislav Petkov | bc8e80d | 2017-06-13 18:28:30 +0200 | [diff] [blame] | 916 | #ifdef CONFIG_CPU_SUP_AMD |
Daniel J Blueman | 8b84c8d | 2012-11-27 14:32:10 +0800 | [diff] [blame] | 917 | extern u16 amd_get_nb_id(int cpu); |
Aravind Gopalakrishnan | cc2749e | 2015-06-15 10:28:15 +0200 | [diff] [blame] | 918 | extern u32 amd_get_nodes_per_socket(void); |
Borislav Petkov | bc8e80d | 2017-06-13 18:28:30 +0200 | [diff] [blame] | 919 | #else |
| 920 | static inline u16 amd_get_nb_id(int cpu) { return 0; } |
| 921 | static inline u32 amd_get_nodes_per_socket(void) { return 0; } |
| 922 | #endif |
Andreas Herrmann | 6a81269 | 2009-09-16 11:33:40 +0200 | [diff] [blame] | 923 | |
Jason Wang | 96e39ac | 2013-07-25 16:54:32 +0800 | [diff] [blame] | 924 | static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves) |
| 925 | { |
| 926 | uint32_t base, eax, signature[3]; |
| 927 | |
| 928 | for (base = 0x40000000; base < 0x40010000; base += 0x100) { |
| 929 | cpuid(base, &eax, &signature[0], &signature[1], &signature[2]); |
| 930 | |
| 931 | if (!memcmp(sig, signature, 12) && |
| 932 | (leaves == 0 || ((eax - base) >= leaves))) |
| 933 | return base; |
| 934 | } |
| 935 | |
| 936 | return 0; |
| 937 | } |
| 938 | |
David Howells | f05e798 | 2012-03-28 18:11:12 +0100 | [diff] [blame] | 939 | extern unsigned long arch_align_stack(unsigned long sp); |
| 940 | extern void free_init_pages(char *what, unsigned long begin, unsigned long end); |
| 941 | |
| 942 | void default_idle(void); |
Len Brown | 6a377dd | 2013-02-09 23:08:07 -0500 | [diff] [blame] | 943 | #ifdef CONFIG_XEN |
| 944 | bool xen_set_default_idle(void); |
| 945 | #else |
| 946 | #define xen_set_default_idle 0 |
| 947 | #endif |
David Howells | f05e798 | 2012-03-28 18:11:12 +0100 | [diff] [blame] | 948 | |
| 949 | void stop_this_cpu(void *dummy); |
Borislav Petkov | 4d067d8 | 2013-05-09 12:02:29 +0200 | [diff] [blame] | 950 | void df_debug(struct pt_regs *regs, long error_code); |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 951 | #endif /* _ASM_X86_PROCESSOR_H */ |