blob: cd415b733c2a0595ca8c58c22f23ffb5b5126bc4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __LINUX_CPUMASK_H
2#define __LINUX_CPUMASK_H
3
4/*
5 * Cpumasks provide a bitmap suitable for representing the
Rusty Russell6ba2ef72009-09-24 09:34:53 -06006 * set of CPU's in a system, one bit position per CPU number. In general,
7 * only nr_cpu_ids (<= NR_CPUS) bits are valid.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/kernel.h>
10#include <linux/threads.h>
11#include <linux/bitmap.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050012#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013
Rusty Russellcdfdef72015-03-05 10:49:19 +103014/* Don't assign or return these: may not be this big! */
Rusty Russell2d3854a2008-11-05 13:39:10 +110015typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
Rusty Russellae7a47e2008-12-30 09:05:15 +103017/**
Rusty Russell6ba2ef72009-09-24 09:34:53 -060018 * cpumask_bits - get the bits in a cpumask
19 * @maskp: the struct cpumask *
Rusty Russellae7a47e2008-12-30 09:05:15 +103020 *
Rusty Russell6ba2ef72009-09-24 09:34:53 -060021 * You should only assume nr_cpu_ids bits of this mask are valid. This is
22 * a macro so it's const-correct.
Rusty Russellae7a47e2008-12-30 09:05:15 +103023 */
Rusty Russell6ba2ef72009-09-24 09:34:53 -060024#define cpumask_bits(maskp) ((maskp)->bits)
Paul Jackson7ea931c2008-04-28 02:12:29 -070025
Tejun Heof1bbc032015-02-13 14:36:57 -080026/**
27 * cpumask_pr_args - printf args to output a cpumask
28 * @maskp: cpumask to be printed
29 *
30 * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
31 */
32#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
33
Mike Travis41df0d612008-05-12 21:21:13 +020034#if NR_CPUS == 1
Alexey Dobriyan9b130ad2017-09-08 16:14:18 -070035#define nr_cpu_ids 1U
Rusty Russell6ba2ef72009-09-24 09:34:53 -060036#else
Alexey Dobriyan9b130ad2017-09-08 16:14:18 -070037extern unsigned int nr_cpu_ids;
Mike Travis41df0d612008-05-12 21:21:13 +020038#endif
39
Rusty Russell6ba2ef72009-09-24 09:34:53 -060040#ifdef CONFIG_CPUMASK_OFFSTACK
41/* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also,
42 * not all bits may be allocated. */
Alexey Dobriyan9b130ad2017-09-08 16:14:18 -070043#define nr_cpumask_bits nr_cpu_ids
Rusty Russell6ba2ef72009-09-24 09:34:53 -060044#else
Alexey Dobriyanc311c792017-05-08 15:56:15 -070045#define nr_cpumask_bits ((unsigned int)NR_CPUS)
Rusty Russell6ba2ef72009-09-24 09:34:53 -060046#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
48/*
49 * The following particular system cpumasks and operations manage
Rusty Russellb3199c02008-12-30 09:05:14 +103050 * possible, present, active and online cpus.
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 *
Rusty Russellb3199c02008-12-30 09:05:14 +103052 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
53 * cpu_present_mask - has bit 'cpu' set iff cpu is populated
54 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
55 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 *
Rusty Russellb3199c02008-12-30 09:05:14 +103057 * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 *
Rusty Russellb3199c02008-12-30 09:05:14 +103059 * The cpu_possible_mask is fixed at boot time, as the set of CPU id's
60 * that it is possible might ever be plugged in at anytime during the
61 * life of that system boot. The cpu_present_mask is dynamic(*),
62 * representing which CPUs are currently plugged in. And
63 * cpu_online_mask is the dynamic subset of cpu_present_mask,
64 * indicating those CPUs available for scheduling.
65 *
66 * If HOTPLUG is enabled, then cpu_possible_mask is forced to have
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 * all NR_CPUS bits set, otherwise it is just the set of CPUs that
68 * ACPI reports present at boot.
69 *
Rusty Russellb3199c02008-12-30 09:05:14 +103070 * If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
Linus Torvalds1da177e2005-04-16 15:20:36 -070071 * depending on what ACPI reports as currently plugged in, otherwise
Rusty Russellb3199c02008-12-30 09:05:14 +103072 * cpu_present_mask is just a copy of cpu_possible_mask.
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 *
Rusty Russellb3199c02008-12-30 09:05:14 +103074 * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
75 * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
Linus Torvalds1da177e2005-04-16 15:20:36 -070076 *
77 * Subtleties:
78 * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
79 * assumption that their single CPU is online. The UP
Rusty Russellb3199c02008-12-30 09:05:14 +103080 * cpu_{online,possible,present}_masks are placebos. Changing them
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 * will have no useful affect on the following num_*_cpus()
82 * and cpu_*() macros in the UP case. This ugliness is a UP
83 * optimization - don't waste any instructions or memory references
84 * asking if you're online or how many CPUs there are if there is
85 * only one CPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -070086 */
87
Rasmus Villemoes4b804c82016-01-20 15:00:19 -080088extern struct cpumask __cpu_possible_mask;
89extern struct cpumask __cpu_online_mask;
90extern struct cpumask __cpu_present_mask;
91extern struct cpumask __cpu_active_mask;
Rasmus Villemoes5aec01b2016-01-20 15:00:25 -080092#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
93#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
94#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
95#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
Rusty Russellb3199c02008-12-30 09:05:14 +103096
Linus Torvalds1da177e2005-04-16 15:20:36 -070097#if NR_CPUS > 1
Rusty Russellae7a47e2008-12-30 09:05:15 +103098#define num_online_cpus() cpumask_weight(cpu_online_mask)
99#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
100#define num_present_cpus() cpumask_weight(cpu_present_mask)
Peter Zijlstra6ad4c182009-11-25 13:31:39 +0100101#define num_active_cpus() cpumask_weight(cpu_active_mask)
Rusty Russellae7a47e2008-12-30 09:05:15 +1030102#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
103#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
104#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
105#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106#else
Heiko Carstens221e3eb2010-03-05 13:42:41 -0800107#define num_online_cpus() 1U
108#define num_possible_cpus() 1U
109#define num_present_cpus() 1U
110#define num_active_cpus() 1U
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#define cpu_online(cpu) ((cpu) == 0)
112#define cpu_possible(cpu) ((cpu) == 0)
113#define cpu_present(cpu) ((cpu) == 0)
Max Krasnyanskye761b772008-07-15 04:43:49 -0700114#define cpu_active(cpu) ((cpu) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115#endif
116
Rusty Russell2d3854a2008-11-05 13:39:10 +1100117/* verify cpu argument to cpumask_* operators */
118static inline unsigned int cpumask_check(unsigned int cpu)
119{
120#ifdef CONFIG_DEBUG_PER_CPU_MAPS
121 WARN_ON_ONCE(cpu >= nr_cpumask_bits);
122#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
123 return cpu;
124}
125
126#if NR_CPUS == 1
Rusty Russell984f2f32008-11-08 20:24:19 +1100127/* Uniprocessor. Assume all masks are "1". */
128static inline unsigned int cpumask_first(const struct cpumask *srcp)
129{
130 return 0;
131}
132
133/* Valid inputs for n are -1 and 0. */
134static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
135{
136 return n+1;
137}
138
139static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
140{
141 return n+1;
142}
143
144static inline unsigned int cpumask_next_and(int n,
145 const struct cpumask *srcp,
146 const struct cpumask *andp)
147{
148 return n+1;
149}
150
151/* cpu must be a valid cpu, ie 0, so there's no other choice. */
152static inline unsigned int cpumask_any_but(const struct cpumask *mask,
153 unsigned int cpu)
154{
155 return 1;
156}
Rusty Russell2d3854a2008-11-05 13:39:10 +1100157
Rusty Russellf36963c2015-05-09 03:14:13 +0930158static inline unsigned int cpumask_local_spread(unsigned int i, int node)
Amir Vadaida913092014-06-09 10:24:38 +0300159{
Amir Vadaida913092014-06-09 10:24:38 +0300160 return 0;
161}
162
Rusty Russell2d3854a2008-11-05 13:39:10 +1100163#define for_each_cpu(cpu, mask) \
164 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
Paul E. McKenney8bd93a22010-02-22 17:04:59 -0800165#define for_each_cpu_not(cpu, mask) \
166 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
Rusty Russell2d3854a2008-11-05 13:39:10 +1100167#define for_each_cpu_and(cpu, mask, and) \
168 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
169#else
170/**
171 * cpumask_first - get the first cpu in a cpumask
172 * @srcp: the cpumask pointer
173 *
174 * Returns >= nr_cpu_ids if no cpus set.
175 */
176static inline unsigned int cpumask_first(const struct cpumask *srcp)
177{
178 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
179}
180
Alexey Dobriyanf22ef332017-09-08 16:17:15 -0700181unsigned int cpumask_next(int n, const struct cpumask *srcp);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100182
183/**
184 * cpumask_next_zero - get the next unset cpu in a cpumask
185 * @n: the cpu prior to the place to search (ie. return will be > @n)
186 * @srcp: the cpumask pointer
187 *
188 * Returns >= nr_cpu_ids if no further cpus unset.
189 */
190static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
191{
192 /* -1 is a legal arg here. */
193 if (n != -1)
194 cpumask_check(n);
195 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
196}
197
198int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
199int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
Rusty Russellf36963c2015-05-09 03:14:13 +0930200unsigned int cpumask_local_spread(unsigned int i, int node);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100201
Rusty Russell984f2f32008-11-08 20:24:19 +1100202/**
203 * for_each_cpu - iterate over every cpu in a mask
204 * @cpu: the (optionally unsigned) integer iterator
205 * @mask: the cpumask pointer
206 *
207 * After the loop, cpu is >= nr_cpu_ids.
208 */
Rusty Russell2d3854a2008-11-05 13:39:10 +1100209#define for_each_cpu(cpu, mask) \
210 for ((cpu) = -1; \
211 (cpu) = cpumask_next((cpu), (mask)), \
212 (cpu) < nr_cpu_ids;)
Rusty Russell984f2f32008-11-08 20:24:19 +1100213
214/**
Paul E. McKenney8bd93a22010-02-22 17:04:59 -0800215 * for_each_cpu_not - iterate over every cpu in a complemented mask
216 * @cpu: the (optionally unsigned) integer iterator
217 * @mask: the cpumask pointer
218 *
219 * After the loop, cpu is >= nr_cpu_ids.
220 */
221#define for_each_cpu_not(cpu, mask) \
222 for ((cpu) = -1; \
223 (cpu) = cpumask_next_zero((cpu), (mask)), \
224 (cpu) < nr_cpu_ids;)
225
Peter Zijlstrac743f0a2017-04-14 14:20:05 +0200226extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
227
228/**
229 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
230 * @cpu: the (optionally unsigned) integer iterator
231 * @mask: the cpumask poiter
232 * @start: the start location
233 *
234 * The implementation does not assume any bit in @mask is set (including @start).
235 *
236 * After the loop, cpu is >= nr_cpu_ids.
237 */
238#define for_each_cpu_wrap(cpu, mask, start) \
239 for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
240 (cpu) < nr_cpumask_bits; \
241 (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
242
Paul E. McKenney8bd93a22010-02-22 17:04:59 -0800243/**
Rusty Russell984f2f32008-11-08 20:24:19 +1100244 * for_each_cpu_and - iterate over every cpu in both masks
245 * @cpu: the (optionally unsigned) integer iterator
246 * @mask: the first cpumask pointer
247 * @and: the second cpumask pointer
248 *
249 * This saves a temporary CPU mask in many places. It is equivalent to:
250 * struct cpumask tmp;
251 * cpumask_and(&tmp, &mask, &and);
252 * for_each_cpu(cpu, &tmp)
253 * ...
254 *
255 * After the loop, cpu is >= nr_cpu_ids.
256 */
Rusty Russell2d3854a2008-11-05 13:39:10 +1100257#define for_each_cpu_and(cpu, mask, and) \
258 for ((cpu) = -1; \
259 (cpu) = cpumask_next_and((cpu), (mask), (and)), \
260 (cpu) < nr_cpu_ids;)
261#endif /* SMP */
262
263#define CPU_BITS_NONE \
264{ \
265 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
266}
267
268#define CPU_BITS_CPU0 \
269{ \
270 [0] = 1UL \
271}
272
273/**
274 * cpumask_set_cpu - set a cpu in a cpumask
275 * @cpu: cpu number (< nr_cpu_ids)
276 * @dstp: the cpumask pointer
277 */
278static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
279{
280 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
281}
282
Peter Zijlstra6c8557b2017-05-19 12:58:25 +0200283static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
284{
285 __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
286}
287
288
Rusty Russell2d3854a2008-11-05 13:39:10 +1100289/**
290 * cpumask_clear_cpu - clear a cpu in a cpumask
291 * @cpu: cpu number (< nr_cpu_ids)
292 * @dstp: the cpumask pointer
293 */
294static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
295{
296 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
297}
298
Peter Zijlstra6c8557b2017-05-19 12:58:25 +0200299static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
300{
301 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
302}
303
Rusty Russell2d3854a2008-11-05 13:39:10 +1100304/**
305 * cpumask_test_cpu - test for a cpu in a cpumask
306 * @cpu: cpu number (< nr_cpu_ids)
307 * @cpumask: the cpumask pointer
308 *
Alex Shic777ad62012-05-28 22:23:51 +0800309 * Returns 1 if @cpu is set in @cpumask, else returns 0
Rusty Russell2d3854a2008-11-05 13:39:10 +1100310 */
Rasmus Villemoes3bbf7f42015-03-31 13:25:05 +1030311static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
312{
313 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
314}
Rusty Russell2d3854a2008-11-05 13:39:10 +1100315
316/**
317 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
318 * @cpu: cpu number (< nr_cpu_ids)
319 * @cpumask: the cpumask pointer
320 *
Alex Shic777ad62012-05-28 22:23:51 +0800321 * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
322 *
Rusty Russell2d3854a2008-11-05 13:39:10 +1100323 * test_and_set_bit wrapper for cpumasks.
324 */
325static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
326{
327 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
328}
329
330/**
Xiao Guangrong54fdade2009-09-22 16:43:39 -0700331 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
332 * @cpu: cpu number (< nr_cpu_ids)
333 * @cpumask: the cpumask pointer
334 *
Alex Shic777ad62012-05-28 22:23:51 +0800335 * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
336 *
Xiao Guangrong54fdade2009-09-22 16:43:39 -0700337 * test_and_clear_bit wrapper for cpumasks.
338 */
339static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
340{
341 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
342}
343
344/**
Rusty Russell2d3854a2008-11-05 13:39:10 +1100345 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
346 * @dstp: the cpumask pointer
347 */
348static inline void cpumask_setall(struct cpumask *dstp)
349{
350 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
351}
352
353/**
354 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
355 * @dstp: the cpumask pointer
356 */
357static inline void cpumask_clear(struct cpumask *dstp)
358{
359 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
360}
361
362/**
363 * cpumask_and - *dstp = *src1p & *src2p
364 * @dstp: the cpumask result
365 * @src1p: the first input
366 * @src2p: the second input
Alex Shic777ad62012-05-28 22:23:51 +0800367 *
368 * If *@dstp is empty, returns 0, else returns 1
Rusty Russell2d3854a2008-11-05 13:39:10 +1100369 */
Linus Torvaldsf4b03732009-08-21 09:26:15 -0700370static inline int cpumask_and(struct cpumask *dstp,
Rusty Russell2d3854a2008-11-05 13:39:10 +1100371 const struct cpumask *src1p,
372 const struct cpumask *src2p)
373{
Linus Torvaldsf4b03732009-08-21 09:26:15 -0700374 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
Rusty Russell2d3854a2008-11-05 13:39:10 +1100375 cpumask_bits(src2p), nr_cpumask_bits);
376}
377
378/**
379 * cpumask_or - *dstp = *src1p | *src2p
380 * @dstp: the cpumask result
381 * @src1p: the first input
382 * @src2p: the second input
383 */
384static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
385 const struct cpumask *src2p)
386{
387 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
388 cpumask_bits(src2p), nr_cpumask_bits);
389}
390
391/**
392 * cpumask_xor - *dstp = *src1p ^ *src2p
393 * @dstp: the cpumask result
394 * @src1p: the first input
395 * @src2p: the second input
396 */
397static inline void cpumask_xor(struct cpumask *dstp,
398 const struct cpumask *src1p,
399 const struct cpumask *src2p)
400{
401 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
402 cpumask_bits(src2p), nr_cpumask_bits);
403}
404
405/**
406 * cpumask_andnot - *dstp = *src1p & ~*src2p
407 * @dstp: the cpumask result
408 * @src1p: the first input
409 * @src2p: the second input
Alex Shic777ad62012-05-28 22:23:51 +0800410 *
411 * If *@dstp is empty, returns 0, else returns 1
Rusty Russell2d3854a2008-11-05 13:39:10 +1100412 */
Linus Torvaldsf4b03732009-08-21 09:26:15 -0700413static inline int cpumask_andnot(struct cpumask *dstp,
Rusty Russell2d3854a2008-11-05 13:39:10 +1100414 const struct cpumask *src1p,
415 const struct cpumask *src2p)
416{
Linus Torvaldsf4b03732009-08-21 09:26:15 -0700417 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
Rusty Russell2d3854a2008-11-05 13:39:10 +1100418 cpumask_bits(src2p), nr_cpumask_bits);
419}
420
421/**
422 * cpumask_complement - *dstp = ~*srcp
423 * @dstp: the cpumask result
424 * @srcp: the input to invert
425 */
426static inline void cpumask_complement(struct cpumask *dstp,
427 const struct cpumask *srcp)
428{
429 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
430 nr_cpumask_bits);
431}
432
433/**
434 * cpumask_equal - *src1p == *src2p
435 * @src1p: the first input
436 * @src2p: the second input
437 */
438static inline bool cpumask_equal(const struct cpumask *src1p,
439 const struct cpumask *src2p)
440{
441 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
442 nr_cpumask_bits);
443}
444
445/**
446 * cpumask_intersects - (*src1p & *src2p) != 0
447 * @src1p: the first input
448 * @src2p: the second input
449 */
450static inline bool cpumask_intersects(const struct cpumask *src1p,
451 const struct cpumask *src2p)
452{
453 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
454 nr_cpumask_bits);
455}
456
457/**
458 * cpumask_subset - (*src1p & ~*src2p) == 0
459 * @src1p: the first input
460 * @src2p: the second input
Alex Shic777ad62012-05-28 22:23:51 +0800461 *
462 * Returns 1 if *@src1p is a subset of *@src2p, else returns 0
Rusty Russell2d3854a2008-11-05 13:39:10 +1100463 */
464static inline int cpumask_subset(const struct cpumask *src1p,
465 const struct cpumask *src2p)
466{
467 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
468 nr_cpumask_bits);
469}
470
471/**
472 * cpumask_empty - *srcp == 0
473 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
474 */
475static inline bool cpumask_empty(const struct cpumask *srcp)
476{
477 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
478}
479
480/**
481 * cpumask_full - *srcp == 0xFFFFFFFF...
482 * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
483 */
484static inline bool cpumask_full(const struct cpumask *srcp)
485{
486 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
487}
488
489/**
490 * cpumask_weight - Count of bits in *srcp
491 * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
492 */
493static inline unsigned int cpumask_weight(const struct cpumask *srcp)
494{
495 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
496}
497
498/**
499 * cpumask_shift_right - *dstp = *srcp >> n
500 * @dstp: the cpumask result
501 * @srcp: the input to shift
502 * @n: the number of bits to shift by
503 */
504static inline void cpumask_shift_right(struct cpumask *dstp,
505 const struct cpumask *srcp, int n)
506{
507 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
508 nr_cpumask_bits);
509}
510
511/**
512 * cpumask_shift_left - *dstp = *srcp << n
513 * @dstp: the cpumask result
514 * @srcp: the input to shift
515 * @n: the number of bits to shift by
516 */
517static inline void cpumask_shift_left(struct cpumask *dstp,
518 const struct cpumask *srcp, int n)
519{
520 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
521 nr_cpumask_bits);
522}
523
524/**
525 * cpumask_copy - *dstp = *srcp
526 * @dstp: the result
527 * @srcp: the input cpumask
528 */
529static inline void cpumask_copy(struct cpumask *dstp,
530 const struct cpumask *srcp)
531{
532 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
533}
534
535/**
536 * cpumask_any - pick a "random" cpu from *srcp
537 * @srcp: the input cpumask
538 *
539 * Returns >= nr_cpu_ids if no cpus set.
540 */
541#define cpumask_any(srcp) cpumask_first(srcp)
542
543/**
544 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
545 * @src1p: the first input
546 * @src2p: the second input
547 *
548 * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
549 */
550#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
551
552/**
553 * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
554 * @mask1: the first input cpumask
555 * @mask2: the second input cpumask
556 *
557 * Returns >= nr_cpu_ids if no cpus set.
558 */
559#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
560
561/**
Rusty Russellcd83e422008-11-07 11:12:29 +1100562 * cpumask_of - the cpumask containing just a given cpu
563 * @cpu: the cpu (<= nr_cpu_ids)
564 */
565#define cpumask_of(cpu) (get_cpu_mask(cpu))
566
567/**
Rusty Russell29c01772008-12-13 21:20:25 +1030568 * cpumask_parse_user - extract a cpumask from a user string
569 * @buf: the buffer to extract from
570 * @len: the length of the buffer
571 * @dstp: the cpumask to set.
572 *
573 * Returns -errno, or 0 for success.
574 */
575static inline int cpumask_parse_user(const char __user *buf, int len,
576 struct cpumask *dstp)
577{
Tejun Heo4d59b6c2017-02-08 14:30:56 -0800578 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
Rusty Russell29c01772008-12-13 21:20:25 +1030579}
580
581/**
Mike Travis4b0604202011-05-24 17:13:12 -0700582 * cpumask_parselist_user - extract a cpumask from a user string
583 * @buf: the buffer to extract from
584 * @len: the length of the buffer
585 * @dstp: the cpumask to set.
586 *
587 * Returns -errno, or 0 for success.
588 */
589static inline int cpumask_parselist_user(const char __user *buf, int len,
590 struct cpumask *dstp)
591{
592 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
Tejun Heo4d59b6c2017-02-08 14:30:56 -0800593 nr_cpumask_bits);
Mike Travis4b0604202011-05-24 17:13:12 -0700594}
595
596/**
Geliang Tangb06fb412016-08-02 14:05:42 -0700597 * cpumask_parse - extract a cpumask from a string
Tejun Heoba630e42013-03-12 11:30:04 -0700598 * @buf: the buffer to extract from
599 * @dstp: the cpumask to set.
600 *
601 * Returns -errno, or 0 for success.
602 */
603static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
604{
605 char *nl = strchr(buf, '\n');
Brian W Hartcea092c2014-05-14 10:33:45 +0930606 unsigned int len = nl ? (unsigned int)(nl - buf) : strlen(buf);
Tejun Heoba630e42013-03-12 11:30:04 -0700607
Tejun Heo4d59b6c2017-02-08 14:30:56 -0800608 return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
Tejun Heoba630e42013-03-12 11:30:04 -0700609}
610
611/**
Alex Shi231daf02012-07-27 09:29:42 +0930612 * cpulist_parse - extract a cpumask from a user string of ranges
Rusty Russell29c01772008-12-13 21:20:25 +1030613 * @buf: the buffer to extract from
Rusty Russell29c01772008-12-13 21:20:25 +1030614 * @dstp: the cpumask to set.
615 *
616 * Returns -errno, or 0 for success.
617 */
618static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
619{
Tejun Heo4d59b6c2017-02-08 14:30:56 -0800620 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100621}
622
623/**
624 * cpumask_size - size to allocate for a 'struct cpumask' in bytes
Rusty Russell2d3854a2008-11-05 13:39:10 +1100625 */
626static inline size_t cpumask_size(void)
627{
Rusty Russellcdfdef72015-03-05 10:49:19 +1030628 return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100629}
630
631/*
632 * cpumask_var_t: struct cpumask for stack usage.
633 *
634 * Oh, the wicked games we play! In order to make kernel coding a
635 * little more difficult, we typedef cpumask_var_t to an array or a
636 * pointer: doing &mask on an array is a noop, so it still works.
637 *
638 * ie.
639 * cpumask_var_t tmpmask;
640 * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
641 * return -ENOMEM;
642 *
643 * ... use 'tmpmask' like a normal struct cpumask * ...
644 *
645 * free_cpumask_var(tmpmask);
KOSAKI Motohiroa64a26e2011-07-26 16:08:45 -0700646 *
647 *
648 * However, one notable exception is there. alloc_cpumask_var() allocates
649 * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
650 * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
651 *
652 * cpumask_var_t tmpmask;
653 * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
654 * return -ENOMEM;
655 *
656 * var = *tmpmask;
657 *
658 * This code makes NR_CPUS length memcopy and brings to a memory corruption.
659 * cpumask_copy() provide safe copy functionality.
Christoph Lameter4ba29682014-08-26 19:12:21 -0500660 *
661 * Note that there is another evil here: If you define a cpumask_var_t
662 * as a percpu variable then the way to obtain the address of the cpumask
663 * structure differently influences what this_cpu_* operation needs to be
664 * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
665 * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
666 * other type of cpumask_var_t implementation is configured.
Waiman Long668802c2017-01-30 12:57:43 -0500667 *
668 * Please also note that __cpumask_var_read_mostly can be used to declare
669 * a cpumask_var_t variable itself (not its content) as read mostly.
Rusty Russell2d3854a2008-11-05 13:39:10 +1100670 */
671#ifdef CONFIG_CPUMASK_OFFSTACK
672typedef struct cpumask *cpumask_var_t;
673
Waiman Long668802c2017-01-30 12:57:43 -0500674#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
675#define __cpumask_var_read_mostly __read_mostly
Christoph Lameter4ba29682014-08-26 19:12:21 -0500676
Mike Travis7b4967c2008-12-19 16:56:37 +1030677bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100678bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
Yinghai Lu0281b5d2009-06-06 14:50:36 -0700679bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
680bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100681void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
682void free_cpumask_var(cpumask_var_t mask);
Rusty Russellcd83e422008-11-07 11:12:29 +1100683void free_bootmem_cpumask_var(cpumask_var_t mask);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100684
Matthias Kaehlckef7e30f02017-04-12 11:20:29 -0700685static inline bool cpumask_available(cpumask_var_t mask)
686{
687 return mask != NULL;
688}
689
Rusty Russell2d3854a2008-11-05 13:39:10 +1100690#else
691typedef struct cpumask cpumask_var_t[1];
692
Christoph Lameter4ba29682014-08-26 19:12:21 -0500693#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
Waiman Long668802c2017-01-30 12:57:43 -0500694#define __cpumask_var_read_mostly
Christoph Lameter4ba29682014-08-26 19:12:21 -0500695
Rusty Russell2d3854a2008-11-05 13:39:10 +1100696static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
697{
698 return true;
699}
700
Mike Travis7b4967c2008-12-19 16:56:37 +1030701static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
702 int node)
703{
704 return true;
705}
706
Yinghai Lu0281b5d2009-06-06 14:50:36 -0700707static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
708{
709 cpumask_clear(*mask);
710 return true;
711}
712
713static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
714 int node)
715{
716 cpumask_clear(*mask);
717 return true;
718}
719
Rusty Russell2d3854a2008-11-05 13:39:10 +1100720static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
721{
722}
723
724static inline void free_cpumask_var(cpumask_var_t mask)
725{
726}
Rusty Russellcd83e422008-11-07 11:12:29 +1100727
728static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
729{
730}
Matthias Kaehlckef7e30f02017-04-12 11:20:29 -0700731
732static inline bool cpumask_available(cpumask_var_t mask)
733{
734 return true;
735}
Rusty Russell2d3854a2008-11-05 13:39:10 +1100736#endif /* CONFIG_CPUMASK_OFFSTACK */
737
Rusty Russell2d3854a2008-11-05 13:39:10 +1100738/* It's common to want to use cpu_all_mask in struct member initializers,
739 * so it has to refer to an address rather than a pointer. */
740extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
741#define cpu_all_mask to_cpumask(cpu_all_bits)
742
743/* First bits of cpu_bit_bitmap are in fact unset. */
744#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
745
Rusty Russellae7a47e2008-12-30 09:05:15 +1030746#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
747#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
748#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
749
Rusty Russell2d3854a2008-11-05 13:39:10 +1100750/* Wrappers for arch boot code to manipulate normally-constant masks */
Rusty Russell3fa41522008-12-30 09:05:16 +1030751void init_cpu_present(const struct cpumask *src);
752void init_cpu_possible(const struct cpumask *src);
753void init_cpu_online(const struct cpumask *src);
Rusty Russell6ba2ef72009-09-24 09:34:53 -0600754
Thomas Gleixner427d77a2016-12-13 19:32:28 +0100755static inline void reset_cpu_possible_mask(void)
756{
757 bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
758}
759
Rasmus Villemoes94256762016-01-20 15:00:28 -0800760static inline void
761set_cpu_possible(unsigned int cpu, bool possible)
762{
763 if (possible)
764 cpumask_set_cpu(cpu, &__cpu_possible_mask);
765 else
766 cpumask_clear_cpu(cpu, &__cpu_possible_mask);
767}
768
769static inline void
770set_cpu_present(unsigned int cpu, bool present)
771{
772 if (present)
773 cpumask_set_cpu(cpu, &__cpu_present_mask);
774 else
775 cpumask_clear_cpu(cpu, &__cpu_present_mask);
776}
777
778static inline void
779set_cpu_online(unsigned int cpu, bool online)
780{
Peter Zijlstra (Intel)e9d867a2016-03-10 12:54:08 +0100781 if (online)
Rasmus Villemoes94256762016-01-20 15:00:28 -0800782 cpumask_set_cpu(cpu, &__cpu_online_mask);
Peter Zijlstra (Intel)e9d867a2016-03-10 12:54:08 +0100783 else
Rasmus Villemoes94256762016-01-20 15:00:28 -0800784 cpumask_clear_cpu(cpu, &__cpu_online_mask);
Rasmus Villemoes94256762016-01-20 15:00:28 -0800785}
786
787static inline void
788set_cpu_active(unsigned int cpu, bool active)
789{
790 if (active)
791 cpumask_set_cpu(cpu, &__cpu_active_mask);
792 else
793 cpumask_clear_cpu(cpu, &__cpu_active_mask);
794}
795
796
Rusty Russell6ba2ef72009-09-24 09:34:53 -0600797/**
798 * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
799 * @bitmap: the bitmap
800 *
801 * There are a few places where cpumask_var_t isn't appropriate and
802 * static cpumasks must be used (eg. very early boot), yet we don't
803 * expose the definition of 'struct cpumask'.
804 *
805 * This does the conversion, and can be used as a constant initializer.
806 */
807#define to_cpumask(bitmap) \
808 ((struct cpumask *)(1 ? (bitmap) \
809 : (void *)sizeof(__check_is_bitmap(bitmap))))
810
811static inline int __check_is_bitmap(const unsigned long *bitmap)
812{
813 return 1;
814}
815
816/*
817 * Special-case data structure for "single bit set only" constant CPU masks.
818 *
819 * We pre-generate all the 64 (or 32) possible bit positions, with enough
820 * padding to the left and the right, and return the constant pointer
821 * appropriately offset.
822 */
823extern const unsigned long
824 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
825
826static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
827{
828 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
829 p -= cpu / BITS_PER_LONG;
830 return to_cpumask(p);
831}
832
833#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
834
835#if NR_CPUS <= BITS_PER_LONG
836#define CPU_BITS_ALL \
837{ \
Rusty Russell9941a382015-03-05 10:49:19 +1030838 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
Rusty Russell6ba2ef72009-09-24 09:34:53 -0600839}
840
841#else /* NR_CPUS > BITS_PER_LONG */
842
843#define CPU_BITS_ALL \
844{ \
845 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
Rusty Russell9941a382015-03-05 10:49:19 +1030846 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
Rusty Russell6ba2ef72009-09-24 09:34:53 -0600847}
848#endif /* NR_CPUS > BITS_PER_LONG */
849
Sudeep Holla5aaba362014-09-30 14:48:22 +0100850/**
851 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
852 * as comma-separated list of cpus or hex values of cpumask
853 * @list: indicates whether the cpumap must be list
854 * @mask: the cpumask to copy
855 * @buf: the buffer to copy into
856 *
857 * Returns the length of the (null-terminated) @buf string, zero if
858 * nothing is copied.
859 */
860static inline ssize_t
861cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
862{
863 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
Tejun Heo513e3d22015-02-13 14:36:50 -0800864 nr_cpu_ids);
Sudeep Holla5aaba362014-09-30 14:48:22 +0100865}
866
Rusty Russell9941a382015-03-05 10:49:19 +1030867#if NR_CPUS <= BITS_PER_LONG
868#define CPU_MASK_ALL \
869(cpumask_t) { { \
870 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
871} }
872#else
873#define CPU_MASK_ALL \
874(cpumask_t) { { \
875 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
876 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
877} }
878#endif /* NR_CPUS > BITS_PER_LONG */
879
880#define CPU_MASK_NONE \
881(cpumask_t) { { \
882 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
883} }
884
Rusty Russell15277812015-04-16 12:33:51 +0930885#define CPU_MASK_CPU0 \
886(cpumask_t) { { \
887 [0] = 1UL \
888} }
889
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890#endif /* __LINUX_CPUMASK_H */