Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 1 | /* |
| 2 | * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong |
| 3 | * because MTRRs can span upto 40 bits (36bits on most modern x86) |
| 4 | */ |
| 5 | #define DEBUG |
| 6 | |
| 7 | #include <linux/module.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | #include <linux/init.h> |
| 9 | #include <linux/slab.h> |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 10 | #include <linux/io.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | #include <linux/mm.h> |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 12 | |
| 13 | #include <asm/processor-flags.h> |
| 14 | #include <asm/cpufeature.h> |
| 15 | #include <asm/tlbflush.h> |
| 16 | #include <asm/system.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <asm/mtrr.h> |
| 18 | #include <asm/msr.h> |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 19 | #include <asm/pat.h> |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 20 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #include "mtrr.h" |
| 22 | |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 23 | struct fixed_range_block { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 24 | int base_msr; /* start address of an MTRR block */ |
| 25 | int ranges; /* number of MTRRs in this block */ |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 26 | }; |
| 27 | |
| 28 | static struct fixed_range_block fixed_range_blocks[] = { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 29 | { MSR_MTRRfix64K_00000, 1 }, /* one 64k MTRR */ |
| 30 | { MSR_MTRRfix16K_80000, 2 }, /* two 16k MTRRs */ |
| 31 | { MSR_MTRRfix4K_C0000, 8 }, /* eight 4k MTRRs */ |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 32 | {} |
| 33 | }; |
| 34 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | static unsigned long smp_changes_mask; |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 36 | static int mtrr_state_set; |
Yinghai Lu | 95ffa24 | 2008-04-29 03:52:33 -0700 | [diff] [blame] | 37 | u64 mtrr_tom2; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 39 | struct mtrr_state_type mtrr_state; |
Sheng Yang | 932d27a | 2008-10-09 16:01:53 +0800 | [diff] [blame] | 40 | EXPORT_SYMBOL_GPL(mtrr_state); |
| 41 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 42 | /* |
Andreas Herrmann | 3ff42da | 2009-03-12 17:39:37 +0100 | [diff] [blame] | 43 | * BIOS is expected to clear MtrrFixDramModEn bit, see for example |
| 44 | * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD |
| 45 | * Opteron Processors" (26094 Rev. 3.30 February 2006), section |
| 46 | * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set |
| 47 | * to 1 during BIOS initalization of the fixed MTRRs, then cleared to |
| 48 | * 0 for operation." |
| 49 | */ |
| 50 | static inline void k8_check_syscfg_dram_mod_en(void) |
| 51 | { |
| 52 | u32 lo, hi; |
| 53 | |
| 54 | if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && |
| 55 | (boot_cpu_data.x86 >= 0x0f))) |
| 56 | return; |
| 57 | |
| 58 | rdmsr(MSR_K8_SYSCFG, lo, hi); |
| 59 | if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) { |
| 60 | printk(KERN_ERR FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]" |
| 61 | " not cleared by BIOS, clearing this bit\n", |
| 62 | smp_processor_id()); |
| 63 | lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY; |
| 64 | mtrr_wrmsr(MSR_K8_SYSCFG, lo, hi); |
| 65 | } |
| 66 | } |
| 67 | |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 68 | /* |
| 69 | * Returns the effective MTRR type for the region |
| 70 | * Error returns: |
| 71 | * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR |
| 72 | * - 0xFF - when MTRR is not enabled |
| 73 | */ |
| 74 | u8 mtrr_type_lookup(u64 start, u64 end) |
| 75 | { |
| 76 | int i; |
| 77 | u64 base, mask; |
| 78 | u8 prev_match, curr_match; |
| 79 | |
| 80 | if (!mtrr_state_set) |
| 81 | return 0xFF; |
| 82 | |
| 83 | if (!mtrr_state.enabled) |
| 84 | return 0xFF; |
| 85 | |
| 86 | /* Make end inclusive end, instead of exclusive */ |
| 87 | end--; |
| 88 | |
| 89 | /* Look in fixed ranges. Just return the type as per start */ |
| 90 | if (mtrr_state.have_fixed && (start < 0x100000)) { |
| 91 | int idx; |
| 92 | |
| 93 | if (start < 0x80000) { |
| 94 | idx = 0; |
| 95 | idx += (start >> 16); |
| 96 | return mtrr_state.fixed_ranges[idx]; |
| 97 | } else if (start < 0xC0000) { |
| 98 | idx = 1 * 8; |
| 99 | idx += ((start - 0x80000) >> 14); |
| 100 | return mtrr_state.fixed_ranges[idx]; |
| 101 | } else if (start < 0x1000000) { |
| 102 | idx = 3 * 8; |
| 103 | idx += ((start - 0xC0000) >> 12); |
| 104 | return mtrr_state.fixed_ranges[idx]; |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | /* |
| 109 | * Look in variable ranges |
| 110 | * Look of multiple ranges matching this address and pick type |
| 111 | * as per MTRR precedence |
| 112 | */ |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 113 | if (!(mtrr_state.enabled & 2)) |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 114 | return mtrr_state.def_type; |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 115 | |
| 116 | prev_match = 0xFF; |
| 117 | for (i = 0; i < num_var_ranges; ++i) { |
| 118 | unsigned short start_state, end_state; |
| 119 | |
| 120 | if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11))) |
| 121 | continue; |
| 122 | |
| 123 | base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) + |
| 124 | (mtrr_state.var_ranges[i].base_lo & PAGE_MASK); |
| 125 | mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) + |
| 126 | (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK); |
| 127 | |
| 128 | start_state = ((start & mask) == (base & mask)); |
| 129 | end_state = ((end & mask) == (base & mask)); |
| 130 | if (start_state != end_state) |
| 131 | return 0xFE; |
| 132 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 133 | if ((start & mask) != (base & mask)) |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 134 | continue; |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 135 | |
| 136 | curr_match = mtrr_state.var_ranges[i].base_lo & 0xff; |
| 137 | if (prev_match == 0xFF) { |
| 138 | prev_match = curr_match; |
| 139 | continue; |
| 140 | } |
| 141 | |
| 142 | if (prev_match == MTRR_TYPE_UNCACHABLE || |
| 143 | curr_match == MTRR_TYPE_UNCACHABLE) { |
| 144 | return MTRR_TYPE_UNCACHABLE; |
| 145 | } |
| 146 | |
| 147 | if ((prev_match == MTRR_TYPE_WRBACK && |
| 148 | curr_match == MTRR_TYPE_WRTHROUGH) || |
| 149 | (prev_match == MTRR_TYPE_WRTHROUGH && |
| 150 | curr_match == MTRR_TYPE_WRBACK)) { |
| 151 | prev_match = MTRR_TYPE_WRTHROUGH; |
| 152 | curr_match = MTRR_TYPE_WRTHROUGH; |
| 153 | } |
| 154 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 155 | if (prev_match != curr_match) |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 156 | return MTRR_TYPE_UNCACHABLE; |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 157 | } |
| 158 | |
Yinghai Lu | 95ffa24 | 2008-04-29 03:52:33 -0700 | [diff] [blame] | 159 | if (mtrr_tom2) { |
| 160 | if (start >= (1ULL<<32) && (end < mtrr_tom2)) |
Yinghai Lu | 35605a1 | 2008-03-24 16:02:01 -0700 | [diff] [blame] | 161 | return MTRR_TYPE_WRBACK; |
| 162 | } |
| 163 | |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 164 | if (prev_match != 0xFF) |
| 165 | return prev_match; |
| 166 | |
| 167 | return mtrr_state.def_type; |
| 168 | } |
| 169 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 170 | /* Get the MSR pair relating to a var range */ |
Yinghai Lu | bf8c481 | 2007-06-20 12:23:39 +0200 | [diff] [blame] | 171 | static void |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 172 | get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr) |
| 173 | { |
| 174 | rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); |
| 175 | rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); |
| 176 | } |
| 177 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 178 | /* Fill the MSR pair relating to a var range */ |
Yinghai Lu | 95ffa24 | 2008-04-29 03:52:33 -0700 | [diff] [blame] | 179 | void fill_mtrr_var_range(unsigned int index, |
| 180 | u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi) |
| 181 | { |
| 182 | struct mtrr_var_range *vr; |
| 183 | |
| 184 | vr = mtrr_state.var_ranges; |
| 185 | |
| 186 | vr[index].base_lo = base_lo; |
| 187 | vr[index].base_hi = base_hi; |
| 188 | vr[index].mask_lo = mask_lo; |
| 189 | vr[index].mask_hi = mask_hi; |
| 190 | } |
| 191 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 192 | static void get_fixed_ranges(mtrr_type *frs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 194 | unsigned int *p = (unsigned int *)frs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 195 | int i; |
| 196 | |
Andreas Herrmann | 3ff42da | 2009-03-12 17:39:37 +0100 | [diff] [blame] | 197 | k8_check_syscfg_dram_mod_en(); |
| 198 | |
Jaswinder Singh Rajput | a036c7a | 2009-05-14 12:10:43 +0530 | [diff] [blame] | 199 | rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | |
| 201 | for (i = 0; i < 2; i++) |
Jaswinder Singh Rajput | 7d9d55e | 2009-05-14 12:15:32 +0530 | [diff] [blame] | 202 | rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 203 | for (i = 0; i < 8; i++) |
Jaswinder Singh Rajput | ba5673f | 2009-05-14 12:29:25 +0530 | [diff] [blame] | 204 | rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | } |
| 206 | |
Bernhard Kaindl | 2b3b483 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 207 | void mtrr_save_fixed_ranges(void *info) |
| 208 | { |
Andrew Morton | 84288ad | 2007-07-01 12:06:48 -0700 | [diff] [blame] | 209 | if (cpu_has_mtrr) |
| 210 | get_fixed_ranges(mtrr_state.fixed_ranges); |
Bernhard Kaindl | 2b3b483 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 211 | } |
| 212 | |
Yinghai Lu | d4c90e3 | 2009-03-13 14:08:49 -0700 | [diff] [blame] | 213 | static unsigned __initdata last_fixed_start; |
| 214 | static unsigned __initdata last_fixed_end; |
| 215 | static mtrr_type __initdata last_fixed_type; |
| 216 | |
| 217 | static void __init print_fixed_last(void) |
| 218 | { |
| 219 | if (!last_fixed_end) |
| 220 | return; |
| 221 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 222 | pr_debug(" %05X-%05X %s\n", last_fixed_start, |
| 223 | last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type)); |
Yinghai Lu | d4c90e3 | 2009-03-13 14:08:49 -0700 | [diff] [blame] | 224 | |
| 225 | last_fixed_end = 0; |
| 226 | } |
| 227 | |
| 228 | static void __init update_fixed_last(unsigned base, unsigned end, |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 229 | mtrr_type type) |
Yinghai Lu | d4c90e3 | 2009-03-13 14:08:49 -0700 | [diff] [blame] | 230 | { |
| 231 | last_fixed_start = base; |
| 232 | last_fixed_end = end; |
| 233 | last_fixed_type = type; |
| 234 | } |
| 235 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 236 | static void __init |
| 237 | print_fixed(unsigned base, unsigned step, const mtrr_type *types) |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 238 | { |
| 239 | unsigned i; |
| 240 | |
Yinghai Lu | d4c90e3 | 2009-03-13 14:08:49 -0700 | [diff] [blame] | 241 | for (i = 0; i < 8; ++i, ++types, base += step) { |
| 242 | if (last_fixed_end == 0) { |
| 243 | update_fixed_last(base, base + step, *types); |
| 244 | continue; |
| 245 | } |
| 246 | if (last_fixed_end == base && last_fixed_type == *types) { |
| 247 | last_fixed_end = base + step; |
| 248 | continue; |
| 249 | } |
| 250 | /* new segments: gap or different type */ |
| 251 | print_fixed_last(); |
| 252 | update_fixed_last(base, base + step, *types); |
| 253 | } |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 254 | } |
| 255 | |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 256 | static void prepare_set(void); |
| 257 | static void post_set(void); |
| 258 | |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 259 | static void __init print_mtrr_state(void) |
| 260 | { |
| 261 | unsigned int i; |
| 262 | int high_width; |
| 263 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 264 | pr_debug("MTRR default type: %s\n", |
| 265 | mtrr_attrib_to_str(mtrr_state.def_type)); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 266 | if (mtrr_state.have_fixed) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 267 | pr_debug("MTRR fixed ranges %sabled:\n", |
| 268 | mtrr_state.enabled & 1 ? "en" : "dis"); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 269 | print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0); |
| 270 | for (i = 0; i < 2; ++i) |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 271 | print_fixed(0x80000 + i * 0x20000, 0x04000, |
| 272 | mtrr_state.fixed_ranges + (i + 1) * 8); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 273 | for (i = 0; i < 8; ++i) |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 274 | print_fixed(0xC0000 + i * 0x08000, 0x01000, |
| 275 | mtrr_state.fixed_ranges + (i + 3) * 8); |
Yinghai Lu | d4c90e3 | 2009-03-13 14:08:49 -0700 | [diff] [blame] | 276 | |
| 277 | /* tail */ |
| 278 | print_fixed_last(); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 279 | } |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 280 | pr_debug("MTRR variable ranges %sabled:\n", |
| 281 | mtrr_state.enabled & 2 ? "en" : "dis"); |
Yinghai Lu | 917a015 | 2009-05-06 21:36:16 -0700 | [diff] [blame] | 282 | if (size_or_mask & 0xffffffffUL) |
| 283 | high_width = ffs(size_or_mask & 0xffffffffUL) - 1; |
| 284 | else |
| 285 | high_width = ffs(size_or_mask>>32) + 32 - 1; |
| 286 | high_width = (high_width - (32 - PAGE_SHIFT) + 3) / 4; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 287 | |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 288 | for (i = 0; i < num_var_ranges; ++i) { |
| 289 | if (mtrr_state.var_ranges[i].mask_lo & (1 << 11)) |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 290 | pr_debug(" %u base %0*X%05X000 mask %0*X%05X000 %s\n", |
| 291 | i, |
| 292 | high_width, |
| 293 | mtrr_state.var_ranges[i].base_hi, |
| 294 | mtrr_state.var_ranges[i].base_lo >> 12, |
| 295 | high_width, |
| 296 | mtrr_state.var_ranges[i].mask_hi, |
| 297 | mtrr_state.var_ranges[i].mask_lo >> 12, |
| 298 | mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff)); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 299 | else |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 300 | pr_debug(" %u disabled\n", i); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 301 | } |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 302 | if (mtrr_tom2) |
| 303 | pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 304 | } |
| 305 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 306 | /* Grab all of the MTRR state for this CPU into *state */ |
Sam Ravnborg | 9ef231a | 2007-07-21 17:10:39 +0200 | [diff] [blame] | 307 | void __init get_mtrr_state(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 308 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 309 | struct mtrr_var_range *vrs; |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 310 | unsigned long flags; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 311 | unsigned lo, dummy; |
| 312 | unsigned int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 313 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 314 | vrs = mtrr_state.var_ranges; |
| 315 | |
Jaswinder Singh Rajput | d9bcc01 | 2009-05-14 12:06:12 +0530 | [diff] [blame] | 316 | rdmsr(MSR_MTRRcap, lo, dummy); |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 317 | mtrr_state.have_fixed = (lo >> 8) & 1; |
| 318 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 319 | for (i = 0; i < num_var_ranges; i++) |
| 320 | get_mtrr_var_range(i, &vrs[i]); |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 321 | if (mtrr_state.have_fixed) |
| 322 | get_fixed_ranges(mtrr_state.fixed_ranges); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 323 | |
Jaswinder Singh Rajput | 5265025 | 2009-05-14 12:35:46 +0530 | [diff] [blame] | 324 | rdmsr(MSR_MTRRdefType, lo, dummy); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 325 | mtrr_state.def_type = (lo & 0xff); |
| 326 | mtrr_state.enabled = (lo & 0xc00) >> 10; |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 327 | |
Yinghai Lu | 35605a1 | 2008-03-24 16:02:01 -0700 | [diff] [blame] | 328 | if (amd_special_default_mtrr()) { |
Thomas Gleixner | 0da72a4 | 2008-04-30 20:11:51 +0200 | [diff] [blame] | 329 | unsigned low, high; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 330 | |
Yinghai Lu | 35605a1 | 2008-03-24 16:02:01 -0700 | [diff] [blame] | 331 | /* TOP_MEM2 */ |
Thomas Gleixner | 0da72a4 | 2008-04-30 20:11:51 +0200 | [diff] [blame] | 332 | rdmsr(MSR_K8_TOP_MEM2, low, high); |
Yinghai Lu | 95ffa24 | 2008-04-29 03:52:33 -0700 | [diff] [blame] | 333 | mtrr_tom2 = high; |
| 334 | mtrr_tom2 <<= 32; |
| 335 | mtrr_tom2 |= low; |
Yinghai Lu | 8004dd9 | 2008-05-12 17:40:39 -0700 | [diff] [blame] | 336 | mtrr_tom2 &= 0xffffff800000ULL; |
Yinghai Lu | 35605a1 | 2008-03-24 16:02:01 -0700 | [diff] [blame] | 337 | } |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 338 | |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 339 | print_mtrr_state(); |
| 340 | |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 341 | mtrr_state_set = 1; |
| 342 | |
| 343 | /* PAT setup for BP. We need to go through sync steps here */ |
| 344 | local_irq_save(flags); |
| 345 | prepare_set(); |
| 346 | |
| 347 | pat_init(); |
| 348 | |
| 349 | post_set(); |
| 350 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 351 | } |
| 352 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 353 | /* Some BIOS's are messed up and don't set all MTRRs the same! */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 354 | void __init mtrr_state_warn(void) |
| 355 | { |
| 356 | unsigned long mask = smp_changes_mask; |
| 357 | |
| 358 | if (!mask) |
| 359 | return; |
| 360 | if (mask & MTRR_CHANGE_MASK_FIXED) |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 361 | pr_warning("mtrr: your CPUs had inconsistent fixed MTRR settings\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 362 | if (mask & MTRR_CHANGE_MASK_VARIABLE) |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 363 | pr_warning("mtrr: your CPUs had inconsistent variable MTRR settings\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | if (mask & MTRR_CHANGE_MASK_DEFTYPE) |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 365 | pr_warning("mtrr: your CPUs had inconsistent MTRRdefType settings\n"); |
| 366 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 367 | printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n"); |
| 368 | printk(KERN_INFO "mtrr: corrected configuration.\n"); |
| 369 | } |
| 370 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 371 | /* |
| 372 | * Doesn't attempt to pass an error out to MTRR users |
| 373 | * because it's quite complicated in some cases and probably not |
| 374 | * worth it because the best error handling is to ignore it. |
| 375 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 376 | void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) |
| 377 | { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 378 | if (wrmsr_safe(msr, a, b) < 0) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 379 | printk(KERN_ERR |
| 380 | "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n", |
| 381 | smp_processor_id(), msr, a, b); |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 382 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 383 | } |
| 384 | |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 385 | /** |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 386 | * set_fixed_range - checks & updates a fixed-range MTRR if it |
| 387 | * differs from the value it should have |
Randy Dunlap | 1d3381e | 2008-03-13 16:59:12 -0700 | [diff] [blame] | 388 | * @msr: MSR address of the MTTR which should be checked and updated |
| 389 | * @changed: pointer which indicates whether the MTRR needed to be changed |
| 390 | * @msrwords: pointer to the MSR values which the MSR should have |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 391 | */ |
Paul Jimenez | 2d2ee8d | 2008-01-30 13:30:31 +0100 | [diff] [blame] | 392 | static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords) |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 393 | { |
| 394 | unsigned lo, hi; |
| 395 | |
| 396 | rdmsr(msr, lo, hi); |
| 397 | |
| 398 | if (lo != msrwords[0] || hi != msrwords[1]) { |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 399 | mtrr_wrmsr(msr, msrwords[0], msrwords[1]); |
Paul Jimenez | 2d2ee8d | 2008-01-30 13:30:31 +0100 | [diff] [blame] | 400 | *changed = true; |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 401 | } |
| 402 | } |
| 403 | |
Randy Dunlap | 1d3381e | 2008-03-13 16:59:12 -0700 | [diff] [blame] | 404 | /** |
| 405 | * generic_get_free_region - Get a free MTRR. |
| 406 | * @base: The starting (base) address of the region. |
| 407 | * @size: The size (in bytes) of the region. |
| 408 | * @replace_reg: mtrr index to be replaced; set to invalid value if none. |
| 409 | * |
| 410 | * Returns: The index of the region on success, else negative on error. |
| 411 | */ |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 412 | int |
| 413 | generic_get_free_region(unsigned long base, unsigned long size, int replace_reg) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 414 | { |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 415 | unsigned long lbase, lsize; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 416 | mtrr_type ltype; |
| 417 | int i, max; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | |
| 419 | max = num_var_ranges; |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 420 | if (replace_reg >= 0 && replace_reg < max) |
| 421 | return replace_reg; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 422 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 423 | for (i = 0; i < max; ++i) { |
| 424 | mtrr_if->get(i, &lbase, &lsize, <ype); |
| 425 | if (lsize == 0) |
| 426 | return i; |
| 427 | } |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 428 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 429 | return -ENOSPC; |
| 430 | } |
| 431 | |
Adrian Bunk | 408b664 | 2005-05-01 08:59:29 -0700 | [diff] [blame] | 432 | static void generic_get_mtrr(unsigned int reg, unsigned long *base, |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 433 | unsigned long *size, mtrr_type *type) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 434 | { |
| 435 | unsigned int mask_lo, mask_hi, base_lo, base_hi; |
Yinghai Lu | 38cc1c3 | 2008-08-21 20:24:24 -0700 | [diff] [blame] | 436 | unsigned int tmp, hi; |
Yinghai Lu | 63516ef | 2009-03-13 12:46:07 -0700 | [diff] [blame] | 437 | int cpu; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 438 | |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 439 | /* |
| 440 | * get_mtrr doesn't need to update mtrr_state, also it could be called |
| 441 | * from any cpu, so try to print it out directly. |
| 442 | */ |
Yinghai Lu | 63516ef | 2009-03-13 12:46:07 -0700 | [diff] [blame] | 443 | cpu = get_cpu(); |
| 444 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 445 | rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi); |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 446 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 447 | if ((mask_lo & 0x800) == 0) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 448 | /* Invalid (i.e. free) range */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 449 | *base = 0; |
| 450 | *size = 0; |
| 451 | *type = 0; |
Yinghai Lu | 63516ef | 2009-03-13 12:46:07 -0700 | [diff] [blame] | 452 | goto out_put_cpu; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 453 | } |
| 454 | |
| 455 | rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi); |
| 456 | |
Yinghai Lu | 63516ef | 2009-03-13 12:46:07 -0700 | [diff] [blame] | 457 | /* Work out the shifted address mask: */ |
Yinghai Lu | 38cc1c3 | 2008-08-21 20:24:24 -0700 | [diff] [blame] | 458 | tmp = mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT; |
| 459 | mask_lo = size_or_mask | tmp; |
Yinghai Lu | 63516ef | 2009-03-13 12:46:07 -0700 | [diff] [blame] | 460 | |
| 461 | /* Expand tmp with high bits to all 1s: */ |
Yinghai Lu | 38cc1c3 | 2008-08-21 20:24:24 -0700 | [diff] [blame] | 462 | hi = fls(tmp); |
| 463 | if (hi > 0) { |
| 464 | tmp |= ~((1<<(hi - 1)) - 1); |
| 465 | |
| 466 | if (tmp != mask_lo) { |
Alan Cox | 942fa3b | 2010-02-08 10:03:17 +0000 | [diff] [blame] | 467 | printk(KERN_WARNING "mtrr: your BIOS has configured an incorrect mask, fixing it.\n"); |
Yinghai Lu | 38cc1c3 | 2008-08-21 20:24:24 -0700 | [diff] [blame] | 468 | mask_lo = tmp; |
| 469 | } |
| 470 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | |
Yinghai Lu | 63516ef | 2009-03-13 12:46:07 -0700 | [diff] [blame] | 472 | /* |
| 473 | * This works correctly if size is a power of two, i.e. a |
| 474 | * contiguous range: |
| 475 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 476 | *size = -mask_lo; |
| 477 | *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT; |
| 478 | *type = base_lo & 0xff; |
Yinghai Lu | 8ad9790 | 2009-03-12 18:43:54 -0700 | [diff] [blame] | 479 | |
Yinghai Lu | 63516ef | 2009-03-13 12:46:07 -0700 | [diff] [blame] | 480 | out_put_cpu: |
| 481 | put_cpu(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 482 | } |
| 483 | |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 484 | /** |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 485 | * set_fixed_ranges - checks & updates the fixed-range MTRRs if they |
| 486 | * differ from the saved set |
Randy Dunlap | 1d3381e | 2008-03-13 16:59:12 -0700 | [diff] [blame] | 487 | * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges() |
Bernhard Kaindl | de938c5 | 2007-05-02 19:27:17 +0200 | [diff] [blame] | 488 | */ |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 489 | static int set_fixed_ranges(mtrr_type *frs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 490 | { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 491 | unsigned long long *saved = (unsigned long long *)frs; |
Paul Jimenez | 2d2ee8d | 2008-01-30 13:30:31 +0100 | [diff] [blame] | 492 | bool changed = false; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 493 | int block = -1, range; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 494 | |
Andreas Herrmann | 3ff42da | 2009-03-12 17:39:37 +0100 | [diff] [blame] | 495 | k8_check_syscfg_dram_mod_en(); |
| 496 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 497 | while (fixed_range_blocks[++block].ranges) { |
| 498 | for (range = 0; range < fixed_range_blocks[block].ranges; range++) |
| 499 | set_fixed_range(fixed_range_blocks[block].base_msr + range, |
| 500 | &changed, (unsigned int *)saved++); |
| 501 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 502 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 503 | return changed; |
| 504 | } |
| 505 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 506 | /* |
| 507 | * Set the MSR pair relating to a var range. |
| 508 | * Returns true if changes are made. |
| 509 | */ |
Paul Jimenez | 2d2ee8d | 2008-01-30 13:30:31 +0100 | [diff] [blame] | 510 | static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 511 | { |
| 512 | unsigned int lo, hi; |
Paul Jimenez | 2d2ee8d | 2008-01-30 13:30:31 +0100 | [diff] [blame] | 513 | bool changed = false; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 514 | |
| 515 | rdmsr(MTRRphysBase_MSR(index), lo, hi); |
| 516 | if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL) |
Siddha, Suresh B | cf94b62 | 2005-04-16 15:25:11 -0700 | [diff] [blame] | 517 | || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) != |
| 518 | (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 519 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 520 | mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); |
Paul Jimenez | 2d2ee8d | 2008-01-30 13:30:31 +0100 | [diff] [blame] | 521 | changed = true; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 522 | } |
| 523 | |
| 524 | rdmsr(MTRRphysMask_MSR(index), lo, hi); |
| 525 | |
| 526 | if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL) |
Siddha, Suresh B | cf94b62 | 2005-04-16 15:25:11 -0700 | [diff] [blame] | 527 | || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) != |
| 528 | (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 529 | mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); |
Paul Jimenez | 2d2ee8d | 2008-01-30 13:30:31 +0100 | [diff] [blame] | 530 | changed = true; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 531 | } |
| 532 | return changed; |
| 533 | } |
| 534 | |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 535 | static u32 deftype_lo, deftype_hi; |
| 536 | |
Randy Dunlap | 1d3381e | 2008-03-13 16:59:12 -0700 | [diff] [blame] | 537 | /** |
| 538 | * set_mtrr_state - Set the MTRR state for this CPU. |
| 539 | * |
| 540 | * NOTE: The CPU must already be in a safe state for MTRR changes. |
| 541 | * RETURNS: 0 if no changes made, else a mask indicating what was changed. |
| 542 | */ |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 543 | static unsigned long set_mtrr_state(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 544 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 545 | unsigned long change_mask = 0; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 546 | unsigned int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 547 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 548 | for (i = 0; i < num_var_ranges; i++) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 549 | if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i])) |
| 550 | change_mask |= MTRR_CHANGE_MASK_VARIABLE; |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 551 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 552 | |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 553 | if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 554 | change_mask |= MTRR_CHANGE_MASK_FIXED; |
| 555 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 556 | /* |
| 557 | * Set_mtrr_restore restores the old value of MTRRdefType, |
| 558 | * so to set it we fiddle with the saved value: |
| 559 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 560 | if ((deftype_lo & 0xff) != mtrr_state.def_type |
| 561 | || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 562 | |
| 563 | deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | |
| 564 | (mtrr_state.enabled << 10); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 565 | change_mask |= MTRR_CHANGE_MASK_DEFTYPE; |
| 566 | } |
| 567 | |
| 568 | return change_mask; |
| 569 | } |
| 570 | |
| 571 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 572 | static unsigned long cr4; |
Thomas Gleixner | 40d6753 | 2009-07-25 18:33:11 +0200 | [diff] [blame^] | 573 | static DEFINE_RAW_SPINLOCK(set_atomicity_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 574 | |
| 575 | /* |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 576 | * Since we are disabling the cache don't allow any interrupts, |
| 577 | * they would run extremely slow and would only increase the pain. |
| 578 | * |
| 579 | * The caller must ensure that local interrupts are disabled and |
| 580 | * are reenabled after post_set() has been called. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 581 | */ |
Josh Triplett | 182daa5 | 2006-09-25 23:32:36 -0700 | [diff] [blame] | 582 | static void prepare_set(void) __acquires(set_atomicity_lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 583 | { |
| 584 | unsigned long cr0; |
| 585 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 586 | /* |
| 587 | * Note that this is not ideal |
| 588 | * since the cache is only flushed/disabled for this CPU while the |
| 589 | * MTRRs are changed, but changing this requires more invasive |
| 590 | * changes to the way the kernel boots |
| 591 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 592 | |
Thomas Gleixner | 40d6753 | 2009-07-25 18:33:11 +0200 | [diff] [blame^] | 593 | raw_spin_lock(&set_atomicity_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 594 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 595 | /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */ |
Dave Jones | 7ebad70 | 2008-01-30 13:30:39 +0100 | [diff] [blame] | 596 | cr0 = read_cr0() | X86_CR0_CD; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 597 | write_cr0(cr0); |
| 598 | wbinvd(); |
| 599 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 600 | /* Save value of CR4 and clear Page Global Enable (bit 7) */ |
| 601 | if (cpu_has_pge) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 602 | cr4 = read_cr4(); |
| 603 | write_cr4(cr4 & ~X86_CR4_PGE); |
| 604 | } |
| 605 | |
| 606 | /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ |
| 607 | __flush_tlb(); |
| 608 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 609 | /* Save MTRR state */ |
Jaswinder Singh Rajput | 5265025 | 2009-05-14 12:35:46 +0530 | [diff] [blame] | 610 | rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 611 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 612 | /* Disable MTRRs, and set the default type to uncached */ |
Jaswinder Singh Rajput | 5265025 | 2009-05-14 12:35:46 +0530 | [diff] [blame] | 613 | mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 614 | } |
| 615 | |
Josh Triplett | 182daa5 | 2006-09-25 23:32:36 -0700 | [diff] [blame] | 616 | static void post_set(void) __releases(set_atomicity_lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 617 | { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 618 | /* Flush TLBs (no need to flush caches - they are disabled) */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 619 | __flush_tlb(); |
| 620 | |
| 621 | /* Intel (P6) standard MTRRs */ |
Jaswinder Singh Rajput | 5265025 | 2009-05-14 12:35:46 +0530 | [diff] [blame] | 622 | mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi); |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 623 | |
| 624 | /* Enable caches */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 625 | write_cr0(read_cr0() & 0xbfffffff); |
| 626 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 627 | /* Restore value of CR4 */ |
| 628 | if (cpu_has_pge) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 629 | write_cr4(cr4); |
Thomas Gleixner | 40d6753 | 2009-07-25 18:33:11 +0200 | [diff] [blame^] | 630 | raw_spin_unlock(&set_atomicity_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 631 | } |
| 632 | |
| 633 | static void generic_set_all(void) |
| 634 | { |
| 635 | unsigned long mask, count; |
| 636 | unsigned long flags; |
| 637 | |
| 638 | local_irq_save(flags); |
| 639 | prepare_set(); |
| 640 | |
| 641 | /* Actually set the state */ |
Jan Beulich | 365bff8 | 2006-12-07 02:14:09 +0100 | [diff] [blame] | 642 | mask = set_mtrr_state(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 643 | |
venkatesh.pallipadi@intel.com | 2e5d9c8 | 2008-03-18 17:00:14 -0700 | [diff] [blame] | 644 | /* also set PAT */ |
| 645 | pat_init(); |
| 646 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 647 | post_set(); |
| 648 | local_irq_restore(flags); |
| 649 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 650 | /* Use the atomic bitops to update the global mask */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 651 | for (count = 0; count < sizeof mask * 8; ++count) { |
| 652 | if (mask & 0x01) |
| 653 | set_bit(count, &smp_changes_mask); |
| 654 | mask >>= 1; |
| 655 | } |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 656 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 657 | } |
| 658 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 659 | /** |
| 660 | * generic_set_mtrr - set variable MTRR register on the local CPU. |
| 661 | * |
| 662 | * @reg: The register to set. |
| 663 | * @base: The base address of the region. |
| 664 | * @size: The size of the region. If this is 0 the region is disabled. |
| 665 | * @type: The type of the region. |
| 666 | * |
| 667 | * Returns nothing. |
| 668 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 669 | static void generic_set_mtrr(unsigned int reg, unsigned long base, |
| 670 | unsigned long size, mtrr_type type) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 671 | { |
| 672 | unsigned long flags; |
Shaohua Li | 3b520b2 | 2005-07-07 17:56:38 -0700 | [diff] [blame] | 673 | struct mtrr_var_range *vr; |
| 674 | |
| 675 | vr = &mtrr_state.var_ranges[reg]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 676 | |
| 677 | local_irq_save(flags); |
| 678 | prepare_set(); |
| 679 | |
| 680 | if (size == 0) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 681 | /* |
| 682 | * The invalid bit is kept in the mask, so we simply |
| 683 | * clear the relevant mask register to disable a range. |
| 684 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 685 | mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0); |
Shaohua Li | 3b520b2 | 2005-07-07 17:56:38 -0700 | [diff] [blame] | 686 | memset(vr, 0, sizeof(struct mtrr_var_range)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 687 | } else { |
Shaohua Li | 3b520b2 | 2005-07-07 17:56:38 -0700 | [diff] [blame] | 688 | vr->base_lo = base << PAGE_SHIFT | type; |
| 689 | vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT); |
| 690 | vr->mask_lo = -size << PAGE_SHIFT | 0x800; |
| 691 | vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT); |
| 692 | |
| 693 | mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi); |
| 694 | mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 695 | } |
| 696 | |
| 697 | post_set(); |
| 698 | local_irq_restore(flags); |
| 699 | } |
| 700 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 701 | int generic_validate_add_page(unsigned long base, unsigned long size, |
| 702 | unsigned int type) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 703 | { |
| 704 | unsigned long lbase, last; |
| 705 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 706 | /* |
| 707 | * For Intel PPro stepping <= 7 |
| 708 | * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF |
| 709 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 710 | if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 && |
| 711 | boot_cpu_data.x86_model == 1 && |
| 712 | boot_cpu_data.x86_mask <= 7) { |
| 713 | if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 714 | pr_warning("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 715 | return -EINVAL; |
| 716 | } |
Andreas Mohr | 9b48341 | 2006-12-07 02:14:00 +0100 | [diff] [blame] | 717 | if (!(base + size < 0x70000 || base > 0x7003F) && |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 718 | (type == MTRR_TYPE_WRCOMB |
| 719 | || type == MTRR_TYPE_WRBACK)) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 720 | pr_warning("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 721 | return -EINVAL; |
| 722 | } |
| 723 | } |
| 724 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 725 | /* |
| 726 | * Check upper bits of base and last are equal and lower bits are 0 |
| 727 | * for base and 1 for last |
| 728 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 729 | last = base + size - 1; |
| 730 | for (lbase = base; !(lbase & 1) && (last & 1); |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 731 | lbase = lbase >> 1, last = last >> 1) |
| 732 | ; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 733 | if (lbase != last) { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 734 | pr_warning("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 735 | return -EINVAL; |
| 736 | } |
| 737 | return 0; |
| 738 | } |
| 739 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 740 | static int generic_have_wrcomb(void) |
| 741 | { |
| 742 | unsigned long config, dummy; |
Jaswinder Singh Rajput | d9bcc01 | 2009-05-14 12:06:12 +0530 | [diff] [blame] | 743 | rdmsr(MSR_MTRRcap, config, dummy); |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 744 | return config & (1 << 10); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 745 | } |
| 746 | |
| 747 | int positive_have_wrcomb(void) |
| 748 | { |
| 749 | return 1; |
| 750 | } |
| 751 | |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 752 | /* |
| 753 | * Generic structure... |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 754 | */ |
Emese Revfy | 3b9cfc0 | 2010-01-31 20:16:34 +0100 | [diff] [blame] | 755 | const struct mtrr_ops generic_mtrr_ops = { |
Jaswinder Singh Rajput | a1a499a | 2009-07-04 07:53:00 +0530 | [diff] [blame] | 756 | .use_intel_if = 1, |
| 757 | .set_all = generic_set_all, |
| 758 | .get = generic_get_mtrr, |
| 759 | .get_free_region = generic_get_free_region, |
| 760 | .set = generic_set_mtrr, |
| 761 | .validate_add_page = generic_validate_add_page, |
| 762 | .have_wrcomb = generic_have_wrcomb, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 763 | }; |