Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 2 | #ifndef _ASM_X86_PERCPU_H |
| 3 | #define _ASM_X86_PERCPU_H |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 4 | |
Tejun Heo | 1a51e3a | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 5 | #ifdef CONFIG_X86_64 |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 6 | #define __percpu_seg gs |
| 7 | #define __percpu_mov_op movq |
Tejun Heo | 1a51e3a | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 8 | #else |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 9 | #define __percpu_seg fs |
| 10 | #define __percpu_mov_op movl |
Tejun Heo | 1a51e3a | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 11 | #endif |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 12 | |
| 13 | #ifdef __ASSEMBLY__ |
| 14 | |
| 15 | /* |
| 16 | * PER_CPU finds an address of a per-cpu variable. |
| 17 | * |
| 18 | * Args: |
| 19 | * var - variable name |
| 20 | * reg - 32bit register |
| 21 | * |
| 22 | * The resulting address is stored in the "reg" argument. |
| 23 | * |
| 24 | * Example: |
| 25 | * PER_CPU(cpu_gdt_descr, %ebx) |
| 26 | */ |
| 27 | #ifdef CONFIG_SMP |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 28 | #define PER_CPU(var, reg) \ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 29 | __percpu_mov_op %__percpu_seg:this_cpu_off, reg; \ |
| 30 | lea var(reg), reg |
| 31 | #define PER_CPU_VAR(var) %__percpu_seg:var |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 32 | #else /* ! SMP */ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 33 | #define PER_CPU(var, reg) __percpu_mov_op $var, reg |
| 34 | #define PER_CPU_VAR(var) var |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 35 | #endif /* SMP */ |
| 36 | |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 37 | #ifdef CONFIG_X86_64_SMP |
| 38 | #define INIT_PER_CPU_VAR(var) init_per_cpu__##var |
| 39 | #else |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 40 | #define INIT_PER_CPU_VAR(var) var |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 41 | #endif |
| 42 | |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 43 | #else /* ...!ASSEMBLY */ |
| 44 | |
Tejun Heo | e59a1bb | 2009-06-22 11:56:24 +0900 | [diff] [blame] | 45 | #include <linux/kernel.h> |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 46 | #include <linux/stringify.h> |
| 47 | |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 48 | #ifdef CONFIG_SMP |
Christoph Lameter | d7c3f8c | 2011-03-26 20:57:18 -0500 | [diff] [blame] | 49 | #define __percpu_prefix "%%"__stringify(__percpu_seg)":" |
Alex Shi | c6ae41e | 2012-05-11 15:35:27 +0800 | [diff] [blame] | 50 | #define __my_cpu_offset this_cpu_read(this_cpu_off) |
Brian Gerst | db7829c | 2010-09-09 18:17:26 +0200 | [diff] [blame] | 51 | |
| 52 | /* |
| 53 | * Compared to the generic __my_cpu_offset version, the following |
| 54 | * saves one instruction and avoids clobbering a temp register. |
| 55 | */ |
Tejun Heo | bbc344e | 2014-06-17 19:12:34 -0400 | [diff] [blame] | 56 | #define arch_raw_cpu_ptr(ptr) \ |
Brian Gerst | db7829c | 2010-09-09 18:17:26 +0200 | [diff] [blame] | 57 | ({ \ |
| 58 | unsigned long tcp_ptr__; \ |
Brian Gerst | db7829c | 2010-09-09 18:17:26 +0200 | [diff] [blame] | 59 | asm volatile("add " __percpu_arg(1) ", %0" \ |
| 60 | : "=r" (tcp_ptr__) \ |
| 61 | : "m" (this_cpu_off), "0" (ptr)); \ |
| 62 | (typeof(*(ptr)) __kernel __force *)tcp_ptr__; \ |
| 63 | }) |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 64 | #else |
Christoph Lameter | d7c3f8c | 2011-03-26 20:57:18 -0500 | [diff] [blame] | 65 | #define __percpu_prefix "" |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 66 | #endif |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 67 | |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 68 | #define __percpu_arg(x) __percpu_prefix "%" #x |
Christoph Lameter | d7c3f8c | 2011-03-26 20:57:18 -0500 | [diff] [blame] | 69 | |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 70 | /* |
| 71 | * Initialized pointers to per-cpu variables needed for the boot |
| 72 | * processor need to use these macros to get the proper address |
| 73 | * offset from __per_cpu_load on SMP. |
| 74 | * |
| 75 | * There also must be an entry in vmlinux_64.lds.S |
| 76 | */ |
| 77 | #define DECLARE_INIT_PER_CPU(var) \ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 78 | extern typeof(var) init_per_cpu_var(var) |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 79 | |
| 80 | #ifdef CONFIG_X86_64_SMP |
| 81 | #define init_per_cpu_var(var) init_per_cpu__##var |
| 82 | #else |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 83 | #define init_per_cpu_var(var) var |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 84 | #endif |
| 85 | |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 86 | /* For arch-specific code, we can use direct single-insn ops (they |
| 87 | * don't give an lvalue though). */ |
| 88 | extern void __bad_percpu_size(void); |
| 89 | |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 90 | #define percpu_to_op(op, var, val) \ |
| 91 | do { \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 92 | typedef typeof(var) pto_T__; \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 93 | if (0) { \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 94 | pto_T__ pto_tmp__; \ |
| 95 | pto_tmp__ = (val); \ |
Andi Kleen | 23b764d | 2010-06-10 13:10:36 +0200 | [diff] [blame] | 96 | (void)pto_tmp__; \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 97 | } \ |
| 98 | switch (sizeof(var)) { \ |
| 99 | case 1: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 100 | asm(op "b %1,"__percpu_arg(0) \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 101 | : "+m" (var) \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 102 | : "qi" ((pto_T__)(val))); \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 103 | break; \ |
| 104 | case 2: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 105 | asm(op "w %1,"__percpu_arg(0) \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 106 | : "+m" (var) \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 107 | : "ri" ((pto_T__)(val))); \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 108 | break; \ |
| 109 | case 4: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 110 | asm(op "l %1,"__percpu_arg(0) \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 111 | : "+m" (var) \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 112 | : "ri" ((pto_T__)(val))); \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 113 | break; \ |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 114 | case 8: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 115 | asm(op "q %1,"__percpu_arg(0) \ |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 116 | : "+m" (var) \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 117 | : "re" ((pto_T__)(val))); \ |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 118 | break; \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 119 | default: __bad_percpu_size(); \ |
| 120 | } \ |
| 121 | } while (0) |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 122 | |
Christoph Lameter | 5917dae | 2010-01-05 15:34:50 +0900 | [diff] [blame] | 123 | /* |
| 124 | * Generate a percpu add to memory instruction and optimize code |
Justin P. Mattock | 40f0a5d | 2010-04-19 11:51:16 -0700 | [diff] [blame] | 125 | * if one is added or subtracted. |
Christoph Lameter | 5917dae | 2010-01-05 15:34:50 +0900 | [diff] [blame] | 126 | */ |
| 127 | #define percpu_add_op(var, val) \ |
| 128 | do { \ |
| 129 | typedef typeof(var) pao_T__; \ |
| 130 | const int pao_ID__ = (__builtin_constant_p(val) && \ |
Greg Thelen | bd09d9a | 2013-10-30 13:56:20 -0700 | [diff] [blame] | 131 | ((val) == 1 || (val) == -1)) ? \ |
| 132 | (int)(val) : 0; \ |
Christoph Lameter | 5917dae | 2010-01-05 15:34:50 +0900 | [diff] [blame] | 133 | if (0) { \ |
| 134 | pao_T__ pao_tmp__; \ |
| 135 | pao_tmp__ = (val); \ |
Andi Kleen | 23b764d | 2010-06-10 13:10:36 +0200 | [diff] [blame] | 136 | (void)pao_tmp__; \ |
Christoph Lameter | 5917dae | 2010-01-05 15:34:50 +0900 | [diff] [blame] | 137 | } \ |
| 138 | switch (sizeof(var)) { \ |
| 139 | case 1: \ |
| 140 | if (pao_ID__ == 1) \ |
| 141 | asm("incb "__percpu_arg(0) : "+m" (var)); \ |
| 142 | else if (pao_ID__ == -1) \ |
| 143 | asm("decb "__percpu_arg(0) : "+m" (var)); \ |
| 144 | else \ |
| 145 | asm("addb %1, "__percpu_arg(0) \ |
| 146 | : "+m" (var) \ |
| 147 | : "qi" ((pao_T__)(val))); \ |
| 148 | break; \ |
| 149 | case 2: \ |
| 150 | if (pao_ID__ == 1) \ |
| 151 | asm("incw "__percpu_arg(0) : "+m" (var)); \ |
| 152 | else if (pao_ID__ == -1) \ |
| 153 | asm("decw "__percpu_arg(0) : "+m" (var)); \ |
| 154 | else \ |
| 155 | asm("addw %1, "__percpu_arg(0) \ |
| 156 | : "+m" (var) \ |
| 157 | : "ri" ((pao_T__)(val))); \ |
| 158 | break; \ |
| 159 | case 4: \ |
| 160 | if (pao_ID__ == 1) \ |
| 161 | asm("incl "__percpu_arg(0) : "+m" (var)); \ |
| 162 | else if (pao_ID__ == -1) \ |
| 163 | asm("decl "__percpu_arg(0) : "+m" (var)); \ |
| 164 | else \ |
| 165 | asm("addl %1, "__percpu_arg(0) \ |
| 166 | : "+m" (var) \ |
| 167 | : "ri" ((pao_T__)(val))); \ |
| 168 | break; \ |
| 169 | case 8: \ |
| 170 | if (pao_ID__ == 1) \ |
| 171 | asm("incq "__percpu_arg(0) : "+m" (var)); \ |
| 172 | else if (pao_ID__ == -1) \ |
| 173 | asm("decq "__percpu_arg(0) : "+m" (var)); \ |
| 174 | else \ |
| 175 | asm("addq %1, "__percpu_arg(0) \ |
| 176 | : "+m" (var) \ |
| 177 | : "re" ((pao_T__)(val))); \ |
| 178 | break; \ |
| 179 | default: __bad_percpu_size(); \ |
| 180 | } \ |
| 181 | } while (0) |
| 182 | |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 183 | #define percpu_from_op(op, var) \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 184 | ({ \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 185 | typeof(var) pfo_ret__; \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 186 | switch (sizeof(var)) { \ |
| 187 | case 1: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 188 | asm(op "b "__percpu_arg(1)",%0" \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 189 | : "=q" (pfo_ret__) \ |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 190 | : "m" (var)); \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 191 | break; \ |
| 192 | case 2: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 193 | asm(op "w "__percpu_arg(1)",%0" \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 194 | : "=r" (pfo_ret__) \ |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 195 | : "m" (var)); \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 196 | break; \ |
| 197 | case 4: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 198 | asm(op "l "__percpu_arg(1)",%0" \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 199 | : "=r" (pfo_ret__) \ |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 200 | : "m" (var)); \ |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 201 | break; \ |
| 202 | case 8: \ |
Brian Gerst | 87b2640 | 2009-01-19 00:38:59 +0900 | [diff] [blame] | 203 | asm(op "q "__percpu_arg(1)",%0" \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 204 | : "=r" (pfo_ret__) \ |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 205 | : "m" (var)); \ |
| 206 | break; \ |
| 207 | default: __bad_percpu_size(); \ |
| 208 | } \ |
| 209 | pfo_ret__; \ |
| 210 | }) |
| 211 | |
| 212 | #define percpu_stable_op(op, var) \ |
| 213 | ({ \ |
| 214 | typeof(var) pfo_ret__; \ |
| 215 | switch (sizeof(var)) { \ |
| 216 | case 1: \ |
| 217 | asm(op "b "__percpu_arg(P1)",%0" \ |
| 218 | : "=q" (pfo_ret__) \ |
| 219 | : "p" (&(var))); \ |
| 220 | break; \ |
| 221 | case 2: \ |
| 222 | asm(op "w "__percpu_arg(P1)",%0" \ |
| 223 | : "=r" (pfo_ret__) \ |
| 224 | : "p" (&(var))); \ |
| 225 | break; \ |
| 226 | case 4: \ |
| 227 | asm(op "l "__percpu_arg(P1)",%0" \ |
| 228 | : "=r" (pfo_ret__) \ |
| 229 | : "p" (&(var))); \ |
| 230 | break; \ |
| 231 | case 8: \ |
| 232 | asm(op "q "__percpu_arg(P1)",%0" \ |
| 233 | : "=r" (pfo_ret__) \ |
| 234 | : "p" (&(var))); \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 235 | break; \ |
| 236 | default: __bad_percpu_size(); \ |
| 237 | } \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 238 | pfo_ret__; \ |
Joe Perches | bc9e3be | 2008-03-23 01:03:06 -0700 | [diff] [blame] | 239 | }) |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 240 | |
Jan Beulich | 402af0d | 2010-04-21 15:21:51 +0100 | [diff] [blame] | 241 | #define percpu_unary_op(op, var) \ |
| 242 | ({ \ |
| 243 | switch (sizeof(var)) { \ |
| 244 | case 1: \ |
| 245 | asm(op "b "__percpu_arg(0) \ |
| 246 | : "+m" (var)); \ |
| 247 | break; \ |
| 248 | case 2: \ |
| 249 | asm(op "w "__percpu_arg(0) \ |
| 250 | : "+m" (var)); \ |
| 251 | break; \ |
| 252 | case 4: \ |
| 253 | asm(op "l "__percpu_arg(0) \ |
| 254 | : "+m" (var)); \ |
| 255 | break; \ |
| 256 | case 8: \ |
| 257 | asm(op "q "__percpu_arg(0) \ |
| 258 | : "+m" (var)); \ |
| 259 | break; \ |
| 260 | default: __bad_percpu_size(); \ |
| 261 | } \ |
| 262 | }) |
| 263 | |
Linus Torvalds | ed8d9ad | 2009-08-03 14:08:48 +0900 | [diff] [blame] | 264 | /* |
Tejun Heo | 4030477 | 2010-12-17 15:47:04 +0100 | [diff] [blame] | 265 | * Add return operation |
| 266 | */ |
| 267 | #define percpu_add_return_op(var, val) \ |
| 268 | ({ \ |
| 269 | typeof(var) paro_ret__ = val; \ |
| 270 | switch (sizeof(var)) { \ |
| 271 | case 1: \ |
| 272 | asm("xaddb %0, "__percpu_arg(1) \ |
| 273 | : "+q" (paro_ret__), "+m" (var) \ |
| 274 | : : "memory"); \ |
| 275 | break; \ |
| 276 | case 2: \ |
| 277 | asm("xaddw %0, "__percpu_arg(1) \ |
| 278 | : "+r" (paro_ret__), "+m" (var) \ |
| 279 | : : "memory"); \ |
| 280 | break; \ |
| 281 | case 4: \ |
| 282 | asm("xaddl %0, "__percpu_arg(1) \ |
| 283 | : "+r" (paro_ret__), "+m" (var) \ |
| 284 | : : "memory"); \ |
| 285 | break; \ |
| 286 | case 8: \ |
| 287 | asm("xaddq %0, "__percpu_arg(1) \ |
| 288 | : "+re" (paro_ret__), "+m" (var) \ |
| 289 | : : "memory"); \ |
| 290 | break; \ |
| 291 | default: __bad_percpu_size(); \ |
| 292 | } \ |
| 293 | paro_ret__ += val; \ |
| 294 | paro_ret__; \ |
| 295 | }) |
| 296 | |
| 297 | /* |
Christoph Lameter | 8270137 | 2010-12-14 10:28:47 -0600 | [diff] [blame] | 298 | * xchg is implemented using cmpxchg without a lock prefix. xchg is |
| 299 | * expensive due to the implied lock prefix. The processor cannot prefetch |
| 300 | * cachelines if xchg is used. |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 301 | */ |
| 302 | #define percpu_xchg_op(var, nval) \ |
| 303 | ({ \ |
| 304 | typeof(var) pxo_ret__; \ |
| 305 | typeof(var) pxo_new__ = (nval); \ |
| 306 | switch (sizeof(var)) { \ |
| 307 | case 1: \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 308 | asm("\n\tmov "__percpu_arg(1)",%%al" \ |
| 309 | "\n1:\tcmpxchgb %2, "__percpu_arg(1) \ |
Christoph Lameter | 8270137 | 2010-12-14 10:28:47 -0600 | [diff] [blame] | 310 | "\n\tjnz 1b" \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 311 | : "=&a" (pxo_ret__), "+m" (var) \ |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 312 | : "q" (pxo_new__) \ |
| 313 | : "memory"); \ |
| 314 | break; \ |
| 315 | case 2: \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 316 | asm("\n\tmov "__percpu_arg(1)",%%ax" \ |
| 317 | "\n1:\tcmpxchgw %2, "__percpu_arg(1) \ |
Christoph Lameter | 8270137 | 2010-12-14 10:28:47 -0600 | [diff] [blame] | 318 | "\n\tjnz 1b" \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 319 | : "=&a" (pxo_ret__), "+m" (var) \ |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 320 | : "r" (pxo_new__) \ |
| 321 | : "memory"); \ |
| 322 | break; \ |
| 323 | case 4: \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 324 | asm("\n\tmov "__percpu_arg(1)",%%eax" \ |
| 325 | "\n1:\tcmpxchgl %2, "__percpu_arg(1) \ |
Christoph Lameter | 8270137 | 2010-12-14 10:28:47 -0600 | [diff] [blame] | 326 | "\n\tjnz 1b" \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 327 | : "=&a" (pxo_ret__), "+m" (var) \ |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 328 | : "r" (pxo_new__) \ |
| 329 | : "memory"); \ |
| 330 | break; \ |
| 331 | case 8: \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 332 | asm("\n\tmov "__percpu_arg(1)",%%rax" \ |
| 333 | "\n1:\tcmpxchgq %2, "__percpu_arg(1) \ |
Christoph Lameter | 8270137 | 2010-12-14 10:28:47 -0600 | [diff] [blame] | 334 | "\n\tjnz 1b" \ |
Eric Dumazet | 889a7a6 | 2011-01-25 17:31:54 +0100 | [diff] [blame] | 335 | : "=&a" (pxo_ret__), "+m" (var) \ |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 336 | : "r" (pxo_new__) \ |
| 337 | : "memory"); \ |
| 338 | break; \ |
| 339 | default: __bad_percpu_size(); \ |
| 340 | } \ |
| 341 | pxo_ret__; \ |
| 342 | }) |
| 343 | |
| 344 | /* |
| 345 | * cmpxchg has no such implied lock semantics as a result it is much |
| 346 | * more efficient for cpu local operations. |
| 347 | */ |
| 348 | #define percpu_cmpxchg_op(var, oval, nval) \ |
| 349 | ({ \ |
| 350 | typeof(var) pco_ret__; \ |
| 351 | typeof(var) pco_old__ = (oval); \ |
| 352 | typeof(var) pco_new__ = (nval); \ |
| 353 | switch (sizeof(var)) { \ |
| 354 | case 1: \ |
| 355 | asm("cmpxchgb %2, "__percpu_arg(1) \ |
| 356 | : "=a" (pco_ret__), "+m" (var) \ |
| 357 | : "q" (pco_new__), "0" (pco_old__) \ |
| 358 | : "memory"); \ |
| 359 | break; \ |
| 360 | case 2: \ |
| 361 | asm("cmpxchgw %2, "__percpu_arg(1) \ |
| 362 | : "=a" (pco_ret__), "+m" (var) \ |
| 363 | : "r" (pco_new__), "0" (pco_old__) \ |
| 364 | : "memory"); \ |
| 365 | break; \ |
| 366 | case 4: \ |
| 367 | asm("cmpxchgl %2, "__percpu_arg(1) \ |
| 368 | : "=a" (pco_ret__), "+m" (var) \ |
| 369 | : "r" (pco_new__), "0" (pco_old__) \ |
| 370 | : "memory"); \ |
| 371 | break; \ |
| 372 | case 8: \ |
| 373 | asm("cmpxchgq %2, "__percpu_arg(1) \ |
| 374 | : "=a" (pco_ret__), "+m" (var) \ |
| 375 | : "r" (pco_new__), "0" (pco_old__) \ |
| 376 | : "memory"); \ |
| 377 | break; \ |
| 378 | default: __bad_percpu_size(); \ |
| 379 | } \ |
| 380 | pco_ret__; \ |
| 381 | }) |
| 382 | |
| 383 | /* |
Alex Shi | 641b695 | 2012-05-14 14:15:32 -0700 | [diff] [blame] | 384 | * this_cpu_read() makes gcc load the percpu variable every time it is |
Alex Shi | c6ae41e | 2012-05-11 15:35:27 +0800 | [diff] [blame] | 385 | * accessed while this_cpu_read_stable() allows the value to be cached. |
| 386 | * this_cpu_read_stable() is more efficient and can be used if its value |
Linus Torvalds | ed8d9ad | 2009-08-03 14:08:48 +0900 | [diff] [blame] | 387 | * is guaranteed to be valid across cpus. The current users include |
| 388 | * get_current() and get_thread_info() both of which are actually |
| 389 | * per-thread variables implemented as per-cpu variables and thus |
| 390 | * stable for the duration of the respective task. |
| 391 | */ |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 392 | #define this_cpu_read_stable(var) percpu_stable_op("mov", var) |
Tejun Heo | 9939dda | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 393 | |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 394 | #define raw_cpu_read_1(pcp) percpu_from_op("mov", pcp) |
| 395 | #define raw_cpu_read_2(pcp) percpu_from_op("mov", pcp) |
| 396 | #define raw_cpu_read_4(pcp) percpu_from_op("mov", pcp) |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 397 | |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 398 | #define raw_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) |
| 399 | #define raw_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) |
| 400 | #define raw_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) |
| 401 | #define raw_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
| 402 | #define raw_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
| 403 | #define raw_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
| 404 | #define raw_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
| 405 | #define raw_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
| 406 | #define raw_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
| 407 | #define raw_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) |
| 408 | #define raw_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) |
| 409 | #define raw_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) |
| 410 | #define raw_cpu_xchg_1(pcp, val) percpu_xchg_op(pcp, val) |
| 411 | #define raw_cpu_xchg_2(pcp, val) percpu_xchg_op(pcp, val) |
| 412 | #define raw_cpu_xchg_4(pcp, val) percpu_xchg_op(pcp, val) |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 413 | |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 414 | #define this_cpu_read_1(pcp) percpu_from_op("mov", pcp) |
| 415 | #define this_cpu_read_2(pcp) percpu_from_op("mov", pcp) |
| 416 | #define this_cpu_read_4(pcp) percpu_from_op("mov", pcp) |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 417 | #define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) |
| 418 | #define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) |
| 419 | #define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) |
Christoph Lameter | 5917dae | 2010-01-05 15:34:50 +0900 | [diff] [blame] | 420 | #define this_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
| 421 | #define this_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
| 422 | #define this_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 423 | #define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
| 424 | #define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
| 425 | #define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
| 426 | #define this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) |
| 427 | #define this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) |
| 428 | #define this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 429 | #define this_cpu_xchg_1(pcp, nval) percpu_xchg_op(pcp, nval) |
| 430 | #define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval) |
| 431 | #define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval) |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 432 | |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 433 | #define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) |
| 434 | #define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) |
| 435 | #define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) |
| 436 | #define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 437 | #define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 438 | #define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 439 | |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 440 | #define this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) |
| 441 | #define this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) |
| 442 | #define this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) |
Christoph Lameter | 7296e08 | 2010-12-14 10:28:44 -0600 | [diff] [blame] | 443 | #define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 444 | #define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 445 | #define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 446 | |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 447 | #ifdef CONFIG_X86_CMPXCHG64 |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 448 | #define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2) \ |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 449 | ({ \ |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 450 | bool __ret; \ |
| 451 | typeof(pcp1) __o1 = (o1), __n1 = (n1); \ |
| 452 | typeof(pcp2) __o2 = (o2), __n2 = (n2); \ |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 453 | asm volatile("cmpxchg8b "__percpu_arg(1)"\n\tsetz %0\n\t" \ |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 454 | : "=a" (__ret), "+m" (pcp1), "+m" (pcp2), "+d" (__o2) \ |
| 455 | : "b" (__n1), "c" (__n2), "a" (__o1)); \ |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 456 | __ret; \ |
| 457 | }) |
| 458 | |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 459 | #define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 460 | #define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 461 | #endif /* CONFIG_X86_CMPXCHG64 */ |
| 462 | |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 463 | /* |
| 464 | * Per cpu atomic 64 bit operations are only available under 64 bit. |
| 465 | * 32 bit must fall back to generic operations. |
| 466 | */ |
| 467 | #ifdef CONFIG_X86_64 |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 468 | #define raw_cpu_read_8(pcp) percpu_from_op("mov", pcp) |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 469 | #define raw_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
| 470 | #define raw_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
| 471 | #define raw_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
| 472 | #define raw_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
| 473 | #define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) |
| 474 | #define raw_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) |
| 475 | #define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 476 | |
Jan Beulich | 97b67ae | 2014-11-04 08:50:48 +0000 | [diff] [blame] | 477 | #define this_cpu_read_8(pcp) percpu_from_op("mov", pcp) |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 478 | #define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
| 479 | #define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
| 480 | #define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
| 481 | #define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
| 482 | #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) |
| 483 | #define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) |
Christoph Lameter | 2485b64 | 2011-01-11 18:54:53 +0100 | [diff] [blame] | 484 | #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 485 | |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 486 | /* |
| 487 | * Pretty complex macro to generate cmpxchg16 instruction. The instruction |
| 488 | * is not supported on early AMD64 processors so we must be able to emulate |
| 489 | * it in software. The address used in the cmpxchg16 instruction must be |
| 490 | * aligned to a 16 byte boundary. |
| 491 | */ |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 492 | #define percpu_cmpxchg16b_double(pcp1, pcp2, o1, o2, n1, n2) \ |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 493 | ({ \ |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 494 | bool __ret; \ |
| 495 | typeof(pcp1) __o1 = (o1), __n1 = (n1); \ |
| 496 | typeof(pcp2) __o2 = (o2), __n2 = (n2); \ |
| 497 | alternative_io("leaq %P1,%%rsi\n\tcall this_cpu_cmpxchg16b_emu\n\t", \ |
| 498 | "cmpxchg16b " __percpu_arg(1) "\n\tsetz %0\n\t", \ |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 499 | X86_FEATURE_CX16, \ |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 500 | ASM_OUTPUT2("=a" (__ret), "+m" (pcp1), \ |
| 501 | "+m" (pcp2), "+d" (__o2)), \ |
| 502 | "b" (__n1), "c" (__n2), "a" (__o1) : "rsi"); \ |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 503 | __ret; \ |
| 504 | }) |
| 505 | |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 506 | #define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double |
Jan Beulich | cebef5b | 2011-12-14 08:33:25 +0000 | [diff] [blame] | 507 | #define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double |
Christoph Lameter | b9ec40af | 2011-02-28 11:02:24 +0100 | [diff] [blame] | 508 | |
Christoph Lameter | 30ed1a7 | 2009-10-03 19:48:22 +0900 | [diff] [blame] | 509 | #endif |
| 510 | |
H. Peter Anvin | 117780e | 2016-06-08 12:38:38 -0700 | [diff] [blame] | 511 | static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr, |
Christoph Lameter | 349c004 | 2011-03-12 12:50:10 +0100 | [diff] [blame] | 512 | const unsigned long __percpu *addr) |
| 513 | { |
Lance Richardson | 799bc3c | 2016-09-22 10:03:57 -0400 | [diff] [blame] | 514 | unsigned long __percpu *a = |
| 515 | (unsigned long __percpu *)addr + nr / BITS_PER_LONG; |
Christoph Lameter | 349c004 | 2011-03-12 12:50:10 +0100 | [diff] [blame] | 516 | |
Alex Shi | 641b695 | 2012-05-14 14:15:32 -0700 | [diff] [blame] | 517 | #ifdef CONFIG_X86_64 |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 518 | return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0; |
Alex Shi | 641b695 | 2012-05-14 14:15:32 -0700 | [diff] [blame] | 519 | #else |
Christoph Lameter | b3ca1c1 | 2014-04-07 15:39:34 -0700 | [diff] [blame] | 520 | return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0; |
Alex Shi | 641b695 | 2012-05-14 14:15:32 -0700 | [diff] [blame] | 521 | #endif |
Christoph Lameter | 349c004 | 2011-03-12 12:50:10 +0100 | [diff] [blame] | 522 | } |
| 523 | |
H. Peter Anvin | 117780e | 2016-06-08 12:38:38 -0700 | [diff] [blame] | 524 | static inline bool x86_this_cpu_variable_test_bit(int nr, |
Christoph Lameter | 349c004 | 2011-03-12 12:50:10 +0100 | [diff] [blame] | 525 | const unsigned long __percpu *addr) |
| 526 | { |
H. Peter Anvin | 117780e | 2016-06-08 12:38:38 -0700 | [diff] [blame] | 527 | bool oldbit; |
Christoph Lameter | 349c004 | 2011-03-12 12:50:10 +0100 | [diff] [blame] | 528 | |
Uros Bizjak | 3c52b5c | 2017-09-06 17:18:08 +0200 | [diff] [blame] | 529 | asm volatile("bt "__percpu_arg(2)",%1" |
H. Peter Anvin | 64be6d3 | 2016-06-08 12:38:43 -0700 | [diff] [blame] | 530 | CC_SET(c) |
| 531 | : CC_OUT(c) (oldbit) |
Lance Richardson | 799bc3c | 2016-09-22 10:03:57 -0400 | [diff] [blame] | 532 | : "m" (*(unsigned long __percpu *)addr), "Ir" (nr)); |
Christoph Lameter | 349c004 | 2011-03-12 12:50:10 +0100 | [diff] [blame] | 533 | |
| 534 | return oldbit; |
| 535 | } |
| 536 | |
| 537 | #define x86_this_cpu_test_bit(nr, addr) \ |
| 538 | (__builtin_constant_p((nr)) \ |
| 539 | ? x86_this_cpu_constant_test_bit((nr), (addr)) \ |
| 540 | : x86_this_cpu_variable_test_bit((nr), (addr))) |
| 541 | |
| 542 | |
Ingo Molnar | 6dbde35 | 2009-01-15 22:15:53 +0900 | [diff] [blame] | 543 | #include <asm-generic/percpu.h> |
| 544 | |
| 545 | /* We can use this directly for local CPU (faster). */ |
Jan Beulich | 2c773dd | 2014-11-04 08:26:42 +0000 | [diff] [blame] | 546 | DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off); |
Ingo Molnar | 6dbde35 | 2009-01-15 22:15:53 +0900 | [diff] [blame] | 547 | |
travis@sgi.com | 3334052 | 2008-01-30 13:32:53 +0100 | [diff] [blame] | 548 | #endif /* !__ASSEMBLY__ */ |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 549 | |
| 550 | #ifdef CONFIG_SMP |
| 551 | |
| 552 | /* |
| 553 | * Define the "EARLY_PER_CPU" macros. These are used for some per_cpu |
| 554 | * variables that are initialized and accessed before there are per_cpu |
| 555 | * areas allocated. |
| 556 | */ |
| 557 | |
| 558 | #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ |
| 559 | DEFINE_PER_CPU(_type, _name) = _initvalue; \ |
| 560 | __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ |
| 561 | { [0 ... NR_CPUS-1] = _initvalue }; \ |
Marcin Slusarz | c6a92a2 | 2008-08-17 17:50:50 +0200 | [diff] [blame] | 562 | __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 563 | |
Ido Yariv | c35f774 | 2012-06-11 12:56:45 +0300 | [diff] [blame] | 564 | #define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ |
| 565 | DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue; \ |
| 566 | __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ |
| 567 | { [0 ... NR_CPUS-1] = _initvalue }; \ |
| 568 | __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map |
| 569 | |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 570 | #define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ |
| 571 | EXPORT_PER_CPU_SYMBOL(_name) |
| 572 | |
| 573 | #define DECLARE_EARLY_PER_CPU(_type, _name) \ |
| 574 | DECLARE_PER_CPU(_type, _name); \ |
| 575 | extern __typeof__(_type) *_name##_early_ptr; \ |
| 576 | extern __typeof__(_type) _name##_early_map[] |
| 577 | |
Ido Yariv | c35f774 | 2012-06-11 12:56:45 +0300 | [diff] [blame] | 578 | #define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ |
| 579 | DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \ |
| 580 | extern __typeof__(_type) *_name##_early_ptr; \ |
| 581 | extern __typeof__(_type) _name##_early_map[] |
| 582 | |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 583 | #define early_per_cpu_ptr(_name) (_name##_early_ptr) |
| 584 | #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) |
| 585 | #define early_per_cpu(_name, _cpu) \ |
Tejun Heo | f10fcd47 | 2009-01-13 20:41:34 +0900 | [diff] [blame] | 586 | *(early_per_cpu_ptr(_name) ? \ |
| 587 | &early_per_cpu_ptr(_name)[_cpu] : \ |
| 588 | &per_cpu(_name, _cpu)) |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 589 | |
| 590 | #else /* !CONFIG_SMP */ |
| 591 | #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ |
| 592 | DEFINE_PER_CPU(_type, _name) = _initvalue |
| 593 | |
Ido Yariv | c35f774 | 2012-06-11 12:56:45 +0300 | [diff] [blame] | 594 | #define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ |
| 595 | DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue |
| 596 | |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 597 | #define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ |
| 598 | EXPORT_PER_CPU_SYMBOL(_name) |
| 599 | |
| 600 | #define DECLARE_EARLY_PER_CPU(_type, _name) \ |
| 601 | DECLARE_PER_CPU(_type, _name) |
| 602 | |
Ido Yariv | c35f774 | 2012-06-11 12:56:45 +0300 | [diff] [blame] | 603 | #define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ |
| 604 | DECLARE_PER_CPU_READ_MOSTLY(_type, _name) |
| 605 | |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 606 | #define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) |
| 607 | #define early_per_cpu_ptr(_name) NULL |
| 608 | /* no early_per_cpu_map() */ |
| 609 | |
| 610 | #endif /* !CONFIG_SMP */ |
| 611 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 612 | #endif /* _ASM_X86_PERCPU_H */ |