blob: ec64357e1762bb840ab601ccd3bc67baf465fe54 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __LINUX_PERCPU_H
2#define __LINUX_PERCPU_H
Martin Peschke7ff6f082006-09-25 23:31:21 -07003
Robert P. J. Day0a3021f2007-07-15 23:39:57 -07004#include <linux/preempt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/slab.h> /* For kmalloc() */
6#include <linux/smp.h>
Martin Peschke7ff6f082006-09-25 23:31:21 -07007#include <linux/cpumask.h>
Tejun Heo6a242902009-03-06 14:33:58 +09008#include <linux/pfn.h>
Martin Peschke7ff6f082006-09-25 23:31:21 -07009
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <asm/percpu.h>
11
Tejun Heo6a242902009-03-06 14:33:58 +090012/* enough to cover all DEFINE_PER_CPUs in modules */
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020013#ifdef CONFIG_MODULES
Tejun Heo6a242902009-03-06 14:33:58 +090014#define PERCPU_MODULE_RESERVE (8 << 10)
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020015#else
Tejun Heo6a242902009-03-06 14:33:58 +090016#define PERCPU_MODULE_RESERVE 0
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#endif
18
Tejun Heo6a242902009-03-06 14:33:58 +090019#ifndef PERCPU_ENOUGH_ROOM
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020020#define PERCPU_ENOUGH_ROOM \
Tejun Heo6a242902009-03-06 14:33:58 +090021 (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
22 PERCPU_MODULE_RESERVE)
23#endif
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020024
Jan Blunck632bbfe2006-09-25 23:30:53 -070025/*
26 * Must be an lvalue. Since @var must be a simple identifier,
27 * we force a syntax error here if it isn't.
28 */
29#define get_cpu_var(var) (*({ \
Jan Bluncka666ecf2006-10-06 00:43:58 -070030 extern int simple_identifier_##var(void); \
Jan Blunck632bbfe2006-09-25 23:30:53 -070031 preempt_disable(); \
32 &__get_cpu_var(var); }))
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#define put_cpu_var(var) preempt_enable()
34
35#ifdef CONFIG_SMP
36
Tejun Heoe74e3962009-03-30 19:07:44 +090037#ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA
Tejun Heofbf59bc2009-02-20 16:29:08 +090038
Tejun Heo8d408b42009-02-24 11:57:21 +090039/* minimum unit size, also is the maximum supported allocation size */
Tejun Heo6a242902009-03-06 14:33:58 +090040#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10)
Tejun Heo8d408b42009-02-24 11:57:21 +090041
42/*
43 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
Tejun Heo6b19b0c2009-03-06 14:33:59 +090044 * back on the first chunk for dynamic percpu allocation if arch is
45 * manually allocating and mapping it for faster access (as a part of
46 * large page mapping for example).
Tejun Heo8d408b42009-02-24 11:57:21 +090047 *
Tejun Heo6b19b0c2009-03-06 14:33:59 +090048 * The following values give between one and two pages of free space
49 * after typical minimal boot (2-way SMP, single disk and NIC) with
50 * both defconfig and a distro config on x86_64 and 32. More
51 * intelligent way to determine this would be nice.
Tejun Heo8d408b42009-02-24 11:57:21 +090052 */
Tejun Heo6b19b0c2009-03-06 14:33:59 +090053#if BITS_PER_LONG > 32
54#define PERCPU_DYNAMIC_RESERVE (20 << 10)
55#else
56#define PERCPU_DYNAMIC_RESERVE (12 << 10)
57#endif
Tejun Heo8d408b42009-02-24 11:57:21 +090058
Tejun Heofbf59bc2009-02-20 16:29:08 +090059extern void *pcpu_base_addr;
60
Tejun Heo8d408b42009-02-24 11:57:21 +090061typedef struct page * (*pcpu_get_page_fn_t)(unsigned int cpu, int pageno);
Tejun Heod4b95f82009-07-04 08:10:59 +090062typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size);
63typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);
64typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
Tejun Heo8c4bfc62009-07-04 08:10:59 +090065typedef void (*pcpu_fc_map_fn_t)(void *ptr, size_t size, void *addr);
Tejun Heofbf59bc2009-02-20 16:29:08 +090066
Tejun Heo8d408b42009-02-24 11:57:21 +090067extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
Tejun Heoedcb4632009-03-06 14:33:59 +090068 size_t static_size, size_t reserved_size,
Tejun Heo38a6be52009-07-04 08:10:59 +090069 ssize_t dyn_size, size_t unit_size,
70 void *base_addr);
Tejun Heo8d408b42009-02-24 11:57:21 +090071
Tejun Heo66c3a752009-03-10 16:27:48 +090072extern ssize_t __init pcpu_embed_first_chunk(
73 size_t static_size, size_t reserved_size,
Tejun Heo788e5ab2009-07-04 08:10:58 +090074 ssize_t dyn_size);
Tejun Heo66c3a752009-03-10 16:27:48 +090075
Tejun Heod4b95f82009-07-04 08:10:59 +090076extern ssize_t __init pcpu_4k_first_chunk(
77 size_t static_size, size_t reserved_size,
78 pcpu_fc_alloc_fn_t alloc_fn,
79 pcpu_fc_free_fn_t free_fn,
80 pcpu_fc_populate_pte_fn_t populate_pte_fn);
81
Tejun Heo8c4bfc62009-07-04 08:10:59 +090082#ifdef CONFIG_NEED_MULTIPLE_NODES
83extern ssize_t __init pcpu_lpage_first_chunk(
84 size_t static_size, size_t reserved_size,
85 ssize_t dyn_size, size_t lpage_size,
86 pcpu_fc_alloc_fn_t alloc_fn,
87 pcpu_fc_free_fn_t free_fn,
88 pcpu_fc_map_fn_t map_fn);
89
90extern void *pcpu_lpage_remapped(void *kaddr);
91#else
92static inline ssize_t __init pcpu_lpage_first_chunk(
93 size_t static_size, size_t reserved_size,
94 ssize_t dyn_size, size_t lpage_size,
95 pcpu_fc_alloc_fn_t alloc_fn,
96 pcpu_fc_free_fn_t free_fn,
97 pcpu_fc_map_fn_t map_fn)
98{
99 return -EINVAL;
100}
101
102static inline void *pcpu_lpage_remapped(void *kaddr)
103{
104 return NULL;
105}
106#endif
107
Tejun Heofbf59bc2009-02-20 16:29:08 +0900108/*
109 * Use this to get to a cpu's version of the per-cpu object
110 * dynamically allocated. Non-atomic access to the current CPU's
111 * version should probably be combined with get_cpu()/put_cpu().
112 */
113#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
114
Tejun Heoedcb4632009-03-06 14:33:59 +0900115extern void *__alloc_reserved_percpu(size_t size, size_t align);
116
Tejun Heoe74e3962009-03-30 19:07:44 +0900117#else /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */
Tejun Heofbf59bc2009-02-20 16:29:08 +0900118
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119struct percpu_data {
Eric Dumazetb3242152008-02-06 01:37:01 -0800120 void *ptrs[1];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121};
122
Catalin Marinas2e1483c2009-06-11 13:24:13 +0100123/* pointer disguising messes up the kmemleak objects tracking */
124#ifndef CONFIG_DEBUG_KMEMLEAK
Martin Peschke7ff6f082006-09-25 23:31:21 -0700125#define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
Catalin Marinas2e1483c2009-06-11 13:24:13 +0100126#else
127#define __percpu_disguise(pdata) (struct percpu_data *)(pdata)
128#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
Rusty Russellb36128c2009-02-20 16:29:08 +0900130#define per_cpu_ptr(ptr, cpu) \
131({ \
132 struct percpu_data *__p = __percpu_disguise(ptr); \
133 (__typeof__(ptr))__p->ptrs[(cpu)]; \
134})
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135
Tejun Heoe74e3962009-03-30 19:07:44 +0900136#endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */
Tejun Heofbf59bc2009-02-20 16:29:08 +0900137
Tejun Heof2a82052009-02-20 16:29:08 +0900138extern void *__alloc_percpu(size_t size, size_t align);
139extern void free_percpu(void *__pdata);
140
Tejun Heoe74e3962009-03-30 19:07:44 +0900141#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
142extern void __init setup_per_cpu_areas(void);
143#endif
144
Tejun Heof2a82052009-02-20 16:29:08 +0900145#else /* CONFIG_SMP */
146
147#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
148
149static inline void *__alloc_percpu(size_t size, size_t align)
150{
151 /*
152 * Can't easily make larger alignment work with kmalloc. WARN
153 * on it. Larger alignment should only be used for module
154 * percpu sections on SMP for which this path isn't used.
155 */
Tejun Heoe3176032009-02-26 10:54:17 +0900156 WARN_ON_ONCE(align > SMP_CACHE_BYTES);
Ingo Molnard2b02612009-02-25 14:36:45 +0100157 return kzalloc(size, GFP_KERNEL);
Tejun Heof2a82052009-02-20 16:29:08 +0900158}
159
160static inline void free_percpu(void *p)
161{
162 kfree(p);
163}
164
Tejun Heoe74e3962009-03-30 19:07:44 +0900165static inline void __init setup_per_cpu_areas(void) { }
166
Tejun Heof2a82052009-02-20 16:29:08 +0900167#endif /* CONFIG_SMP */
168
169#define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type), \
170 __alignof__(type))
171
Tejun Heo066123a2009-04-10 12:02:40 -0700172/*
173 * Optional methods for optimized non-lvalue per-cpu variable access.
174 *
175 * @var can be a percpu variable or a field of it and its size should
176 * equal char, int or long. percpu_read() evaluates to a lvalue and
177 * all others to void.
178 *
179 * These operations are guaranteed to be atomic w.r.t. preemption.
180 * The generic versions use plain get/put_cpu_var(). Archs are
181 * encouraged to implement single-instruction alternatives which don't
182 * require preemption protection.
183 */
184#ifndef percpu_read
185# define percpu_read(var) \
186 ({ \
187 typeof(per_cpu_var(var)) __tmp_var__; \
188 __tmp_var__ = get_cpu_var(var); \
189 put_cpu_var(var); \
190 __tmp_var__; \
191 })
192#endif
193
194#define __percpu_generic_to_op(var, val, op) \
195do { \
196 get_cpu_var(var) op val; \
197 put_cpu_var(var); \
198} while (0)
199
200#ifndef percpu_write
201# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
202#endif
203
204#ifndef percpu_add
205# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
206#endif
207
208#ifndef percpu_sub
209# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
210#endif
211
212#ifndef percpu_and
213# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
214#endif
215
216#ifndef percpu_or
217# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
218#endif
219
220#ifndef percpu_xor
221# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
222#endif
223
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224#endif /* __LINUX_PERCPU_H */