blob: 932c26845e3e33933710de5bcc7743db5af0737a [file] [log] [blame]
Thomas Gleixner46aeb7e2019-05-28 10:10:27 -07001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * Simple NUMA memory policy for the Linux kernel.
4 *
5 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
Christoph Lameter8bccd852005-10-29 18:16:59 -07006 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
Christoph Lameter8bccd852005-10-29 18:16:59 -070021 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
Christoph Lameter8bccd852005-10-29 18:16:59 -070024 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * preferred Try a specific node first before normal fallback.
David Rientjes00ef2d22013-02-22 16:35:36 -080029 * As a special case NUMA_NO_NODE here means do the allocation
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
Christoph Lameter8bccd852005-10-29 18:16:59 -070033 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
Linus Torvalds1da177e2005-04-16 15:20:36 -070066*/
67
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -070068#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
69
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <linux/mempolicy.h>
71#include <linux/mm.h>
72#include <linux/highmem.h>
73#include <linux/hugetlb.h>
74#include <linux/kernel.h>
75#include <linux/sched.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010076#include <linux/sched/mm.h>
Ingo Molnar6a3827d2017-02-08 18:51:31 +010077#include <linux/sched/numa_balancing.h>
Ingo Molnarf719ff9b2017-02-06 10:57:33 +010078#include <linux/sched/task.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/nodemask.h>
80#include <linux/cpuset.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#include <linux/slab.h>
82#include <linux/string.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040083#include <linux/export.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070084#include <linux/nsproxy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/interrupt.h>
86#include <linux/init.h>
87#include <linux/compat.h>
Otto Ebeling31367462017-11-15 17:38:14 -080088#include <linux/ptrace.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -080089#include <linux/swap.h>
Christoph Lameter1a75a6c2006-01-08 01:01:02 -080090#include <linux/seq_file.h>
91#include <linux/proc_fs.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080092#include <linux/migrate.h>
Hugh Dickins62b61f62009-12-14 17:59:33 -080093#include <linux/ksm.h>
Christoph Lameter95a402c2006-06-23 02:03:53 -070094#include <linux/rmap.h>
David Quigley86c3a762006-06-23 02:04:02 -070095#include <linux/security.h>
Adrian Bunkdbcb0f12007-10-16 01:26:26 -070096#include <linux/syscalls.h>
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -070097#include <linux/ctype.h>
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -080098#include <linux/mm_inline.h>
Lee Schermerhornb24f53a2012-10-25 14:16:32 +020099#include <linux/mmu_notifier.h>
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -0700100#include <linux/printk.h>
Naoya Horiguchic8633792017-09-08 16:11:08 -0700101#include <linux/swapops.h>
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800102
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <asm/tlbflush.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -0800104#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
Nick Piggin62695a82008-10-18 20:26:09 -0700106#include "internal.h"
107
Christoph Lameter38e35862006-01-08 01:01:01 -0800108/* Internal flags */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800109#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
Christoph Lameter38e35862006-01-08 01:01:01 -0800110#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800111
Pekka Enbergfcc234f2006-03-22 00:08:13 -0800112static struct kmem_cache *policy_cache;
113static struct kmem_cache *sn_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115/* Highest zone. An specific allocation for a zone below that is not
116 policied. */
Christoph Lameter62672762007-02-10 01:43:07 -0800117enum zone_type policy_zone = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700119/*
120 * run-time system-wide default policy => local allocation
121 */
H Hartley Sweetene754d792011-10-31 17:09:23 -0700122static struct mempolicy default_policy = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123 .refcnt = ATOMIC_INIT(1), /* never free it */
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700124 .mode = MPOL_PREFERRED,
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700125 .flags = MPOL_F_LOCAL,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126};
127
Mel Gorman5606e382012-11-02 18:19:13 +0000128static struct mempolicy preferred_node_policy[MAX_NUMNODES];
129
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -0700130struct mempolicy *get_task_policy(struct task_struct *p)
Mel Gorman5606e382012-11-02 18:19:13 +0000131{
132 struct mempolicy *pol = p->mempolicy;
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700133 int node;
Mel Gorman5606e382012-11-02 18:19:13 +0000134
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700135 if (pol)
136 return pol;
Mel Gorman5606e382012-11-02 18:19:13 +0000137
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700138 node = numa_node_id();
139 if (node != NUMA_NO_NODE) {
140 pol = &preferred_node_policy[node];
141 /* preferred_node_policy is not initialised early in boot */
142 if (pol->mode)
143 return pol;
Mel Gorman5606e382012-11-02 18:19:13 +0000144 }
145
Oleg Nesterovf15ca782014-10-09 15:27:43 -0700146 return &default_policy;
Mel Gorman5606e382012-11-02 18:19:13 +0000147}
148
David Rientjes37012942008-04-28 02:12:33 -0700149static const struct mempolicy_operations {
150 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
Vlastimil Babka213980c2017-07-06 15:40:06 -0700151 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
David Rientjes37012942008-04-28 02:12:33 -0700152} mpol_ops[MPOL_MAX];
153
David Rientjesf5b087b2008-04-28 02:12:27 -0700154static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
155{
Bob Liu6d556292010-05-24 14:31:59 -0700156 return pol->flags & MPOL_MODE_FLAGS;
David Rientjes4c50bc02008-04-28 02:12:30 -0700157}
158
159static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
160 const nodemask_t *rel)
161{
162 nodemask_t tmp;
163 nodes_fold(tmp, *orig, nodes_weight(*rel));
164 nodes_onto(*ret, tmp, *rel);
David Rientjesf5b087b2008-04-28 02:12:27 -0700165}
166
David Rientjes37012942008-04-28 02:12:33 -0700167static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
168{
169 if (nodes_empty(*nodes))
170 return -EINVAL;
171 pol->v.nodes = *nodes;
172 return 0;
173}
174
175static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
176{
177 if (!nodes)
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700178 pol->flags |= MPOL_F_LOCAL; /* local allocation */
David Rientjes37012942008-04-28 02:12:33 -0700179 else if (nodes_empty(*nodes))
180 return -EINVAL; /* no allowed nodes */
181 else
182 pol->v.preferred_node = first_node(*nodes);
183 return 0;
184}
185
186static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
187{
Zhihui Zhang859f7ef2014-12-18 16:17:09 -0800188 if (nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700189 return -EINVAL;
190 pol->v.nodes = *nodes;
191 return 0;
192}
193
Miao Xie58568d22009-06-16 15:31:49 -0700194/*
195 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
196 * any, for the new policy. mpol_new() has already validated the nodes
197 * parameter with respect to the policy mode and flags. But, we need to
198 * handle an empty nodemask with MPOL_PREFERRED here.
199 *
200 * Must be called holding task's alloc_lock to protect task's mems_allowed
201 * and mempolicy. May also be called holding the mmap_semaphore for write.
202 */
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700203static int mpol_set_nodemask(struct mempolicy *pol,
204 const nodemask_t *nodes, struct nodemask_scratch *nsc)
Miao Xie58568d22009-06-16 15:31:49 -0700205{
Miao Xie58568d22009-06-16 15:31:49 -0700206 int ret;
207
208 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
209 if (pol == NULL)
210 return 0;
Lai Jiangshan01f13bd2012-12-12 13:51:33 -0800211 /* Check N_MEMORY */
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700212 nodes_and(nsc->mask1,
Lai Jiangshan01f13bd2012-12-12 13:51:33 -0800213 cpuset_current_mems_allowed, node_states[N_MEMORY]);
Miao Xie58568d22009-06-16 15:31:49 -0700214
215 VM_BUG_ON(!nodes);
216 if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
217 nodes = NULL; /* explicit local allocation */
218 else {
219 if (pol->flags & MPOL_F_RELATIVE_NODES)
Zhihui Zhang859f7ef2014-12-18 16:17:09 -0800220 mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
Miao Xie58568d22009-06-16 15:31:49 -0700221 else
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700222 nodes_and(nsc->mask2, *nodes, nsc->mask1);
223
Miao Xie58568d22009-06-16 15:31:49 -0700224 if (mpol_store_user_nodemask(pol))
225 pol->w.user_nodemask = *nodes;
226 else
227 pol->w.cpuset_mems_allowed =
228 cpuset_current_mems_allowed;
229 }
230
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700231 if (nodes)
232 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
233 else
234 ret = mpol_ops[pol->mode].create(pol, NULL);
Miao Xie58568d22009-06-16 15:31:49 -0700235 return ret;
236}
237
238/*
239 * This function just creates a new policy, does some check and simple
240 * initialization. You must invoke mpol_set_nodemask() to set nodes.
241 */
David Rientjes028fec42008-04-28 02:12:25 -0700242static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
243 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244{
245 struct mempolicy *policy;
246
David Rientjes028fec42008-04-28 02:12:25 -0700247 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
David Rientjes00ef2d22013-02-22 16:35:36 -0800248 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
Paul Mundt140d5a42007-07-15 23:38:16 -0700249
David Rientjes3e1f06452008-04-28 02:12:34 -0700250 if (mode == MPOL_DEFAULT) {
251 if (nodes && !nodes_empty(*nodes))
David Rientjes37012942008-04-28 02:12:33 -0700252 return ERR_PTR(-EINVAL);
Lee Schermerhornd3a71032012-10-25 14:16:29 +0200253 return NULL;
David Rientjes37012942008-04-28 02:12:33 -0700254 }
David Rientjes3e1f06452008-04-28 02:12:34 -0700255 VM_BUG_ON(!nodes);
256
257 /*
258 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
259 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
260 * All other modes require a valid pointer to a non-empty nodemask.
261 */
262 if (mode == MPOL_PREFERRED) {
263 if (nodes_empty(*nodes)) {
264 if (((flags & MPOL_F_STATIC_NODES) ||
265 (flags & MPOL_F_RELATIVE_NODES)))
266 return ERR_PTR(-EINVAL);
David Rientjes3e1f06452008-04-28 02:12:34 -0700267 }
Peter Zijlstra479e2802012-10-25 14:16:28 +0200268 } else if (mode == MPOL_LOCAL) {
Piotr Kwapulinski8d303e42016-12-12 16:42:49 -0800269 if (!nodes_empty(*nodes) ||
270 (flags & MPOL_F_STATIC_NODES) ||
271 (flags & MPOL_F_RELATIVE_NODES))
Peter Zijlstra479e2802012-10-25 14:16:28 +0200272 return ERR_PTR(-EINVAL);
273 mode = MPOL_PREFERRED;
David Rientjes3e1f06452008-04-28 02:12:34 -0700274 } else if (nodes_empty(*nodes))
275 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
277 if (!policy)
278 return ERR_PTR(-ENOMEM);
279 atomic_set(&policy->refcnt, 1);
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700280 policy->mode = mode;
David Rientjes3e1f06452008-04-28 02:12:34 -0700281 policy->flags = flags;
David Rientjesf5b087b2008-04-28 02:12:27 -0700282
David Rientjes37012942008-04-28 02:12:33 -0700283 return policy;
284}
285
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700286/* Slow path of a mpol destructor. */
287void __mpol_put(struct mempolicy *p)
288{
289 if (!atomic_dec_and_test(&p->refcnt))
290 return;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700291 kmem_cache_free(policy_cache, p);
292}
293
Vlastimil Babka213980c2017-07-06 15:40:06 -0700294static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
David Rientjes37012942008-04-28 02:12:33 -0700295{
296}
297
Vlastimil Babka213980c2017-07-06 15:40:06 -0700298static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
David Rientjes37012942008-04-28 02:12:33 -0700299{
300 nodemask_t tmp;
301
302 if (pol->flags & MPOL_F_STATIC_NODES)
303 nodes_and(tmp, pol->w.user_nodemask, *nodes);
304 else if (pol->flags & MPOL_F_RELATIVE_NODES)
305 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
306 else {
Vlastimil Babka213980c2017-07-06 15:40:06 -0700307 nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed,
308 *nodes);
zhong jiang29b190f2019-06-28 12:06:43 -0700309 pol->w.cpuset_mems_allowed = *nodes;
David Rientjes37012942008-04-28 02:12:33 -0700310 }
311
Miao Xie708c1bb2010-05-24 14:32:07 -0700312 if (nodes_empty(tmp))
313 tmp = *nodes;
314
Vlastimil Babka213980c2017-07-06 15:40:06 -0700315 pol->v.nodes = tmp;
David Rientjes37012942008-04-28 02:12:33 -0700316}
317
318static void mpol_rebind_preferred(struct mempolicy *pol,
Vlastimil Babka213980c2017-07-06 15:40:06 -0700319 const nodemask_t *nodes)
David Rientjes37012942008-04-28 02:12:33 -0700320{
321 nodemask_t tmp;
322
David Rientjes37012942008-04-28 02:12:33 -0700323 if (pol->flags & MPOL_F_STATIC_NODES) {
324 int node = first_node(pol->w.user_nodemask);
325
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700326 if (node_isset(node, *nodes)) {
David Rientjes37012942008-04-28 02:12:33 -0700327 pol->v.preferred_node = node;
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700328 pol->flags &= ~MPOL_F_LOCAL;
329 } else
330 pol->flags |= MPOL_F_LOCAL;
David Rientjes37012942008-04-28 02:12:33 -0700331 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
332 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
333 pol->v.preferred_node = first_node(tmp);
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700334 } else if (!(pol->flags & MPOL_F_LOCAL)) {
David Rientjes37012942008-04-28 02:12:33 -0700335 pol->v.preferred_node = node_remap(pol->v.preferred_node,
336 pol->w.cpuset_mems_allowed,
337 *nodes);
338 pol->w.cpuset_mems_allowed = *nodes;
339 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340}
341
Miao Xie708c1bb2010-05-24 14:32:07 -0700342/*
343 * mpol_rebind_policy - Migrate a policy to a different set of nodes
344 *
Vlastimil Babka213980c2017-07-06 15:40:06 -0700345 * Per-vma policies are protected by mmap_sem. Allocations using per-task
346 * policies are protected by task->mems_allowed_seq to prevent a premature
347 * OOM/allocation failure due to parallel nodemask modification.
Miao Xie708c1bb2010-05-24 14:32:07 -0700348 */
Vlastimil Babka213980c2017-07-06 15:40:06 -0700349static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
David Rientjes1d0d2682008-04-28 02:12:32 -0700350{
David Rientjes1d0d2682008-04-28 02:12:32 -0700351 if (!pol)
352 return;
Vlastimil Babka2e256442019-03-05 15:46:50 -0800353 if (!mpol_store_user_nodemask(pol) && !(pol->flags & MPOL_F_LOCAL) &&
David Rientjes1d0d2682008-04-28 02:12:32 -0700354 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
355 return;
Miao Xie708c1bb2010-05-24 14:32:07 -0700356
Vlastimil Babka213980c2017-07-06 15:40:06 -0700357 mpol_ops[pol->mode].rebind(pol, newmask);
David Rientjes1d0d2682008-04-28 02:12:32 -0700358}
359
360/*
361 * Wrapper for mpol_rebind_policy() that just requires task
362 * pointer, and updates task mempolicy.
Miao Xie58568d22009-06-16 15:31:49 -0700363 *
364 * Called with task's alloc_lock held.
David Rientjes1d0d2682008-04-28 02:12:32 -0700365 */
366
Vlastimil Babka213980c2017-07-06 15:40:06 -0700367void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
David Rientjes1d0d2682008-04-28 02:12:32 -0700368{
Vlastimil Babka213980c2017-07-06 15:40:06 -0700369 mpol_rebind_policy(tsk->mempolicy, new);
David Rientjes1d0d2682008-04-28 02:12:32 -0700370}
371
372/*
373 * Rebind each vma in mm to new nodemask.
374 *
375 * Call holding a reference to mm. Takes mm->mmap_sem during call.
376 */
377
378void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
379{
380 struct vm_area_struct *vma;
381
382 down_write(&mm->mmap_sem);
383 for (vma = mm->mmap; vma; vma = vma->vm_next)
Vlastimil Babka213980c2017-07-06 15:40:06 -0700384 mpol_rebind_policy(vma->vm_policy, new);
David Rientjes1d0d2682008-04-28 02:12:32 -0700385 up_write(&mm->mmap_sem);
386}
387
David Rientjes37012942008-04-28 02:12:33 -0700388static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
389 [MPOL_DEFAULT] = {
390 .rebind = mpol_rebind_default,
391 },
392 [MPOL_INTERLEAVE] = {
393 .create = mpol_new_interleave,
394 .rebind = mpol_rebind_nodemask,
395 },
396 [MPOL_PREFERRED] = {
397 .create = mpol_new_preferred,
398 .rebind = mpol_rebind_preferred,
399 },
400 [MPOL_BIND] = {
401 .create = mpol_new_bind,
402 .rebind = mpol_rebind_nodemask,
403 },
404};
405
Christoph Lameterfc301282006-01-18 17:42:29 -0800406static void migrate_page_add(struct page *page, struct list_head *pagelist,
407 unsigned long flags);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -0800408
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800409struct queue_pages {
410 struct list_head *pagelist;
411 unsigned long flags;
412 nodemask_t *nmask;
413 struct vm_area_struct *prev;
414};
415
Naoya Horiguchi98094942013-09-11 14:22:14 -0700416/*
Naoya Horiguchi88aaa2a2017-09-08 16:10:42 -0700417 * Check if the page's nid is in qp->nmask.
418 *
419 * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
420 * in the invert of qp->nmask.
421 */
422static inline bool queue_pages_required(struct page *page,
423 struct queue_pages *qp)
424{
425 int nid = page_to_nid(page);
426 unsigned long flags = qp->flags;
427
428 return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
429}
430
Yang Shia7f40cf2019-03-28 20:43:55 -0700431/*
Yang Shid8835442019-08-13 15:37:15 -0700432 * queue_pages_pmd() has four possible return values:
433 * 0 - pages are placed on the right node or queued successfully.
434 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
435 * specified.
436 * 2 - THP was split.
437 * -EIO - is migration entry or only MPOL_MF_STRICT was specified and an
438 * existing page was already on a node that does not follow the
439 * policy.
Yang Shia7f40cf2019-03-28 20:43:55 -0700440 */
Naoya Horiguchic8633792017-09-08 16:11:08 -0700441static int queue_pages_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr,
442 unsigned long end, struct mm_walk *walk)
443{
444 int ret = 0;
445 struct page *page;
446 struct queue_pages *qp = walk->private;
447 unsigned long flags;
448
449 if (unlikely(is_pmd_migration_entry(*pmd))) {
Yang Shia7f40cf2019-03-28 20:43:55 -0700450 ret = -EIO;
Naoya Horiguchic8633792017-09-08 16:11:08 -0700451 goto unlock;
452 }
453 page = pmd_page(*pmd);
454 if (is_huge_zero_page(page)) {
455 spin_unlock(ptl);
456 __split_huge_pmd(walk->vma, pmd, addr, false, NULL);
Yang Shid8835442019-08-13 15:37:15 -0700457 ret = 2;
Naoya Horiguchic8633792017-09-08 16:11:08 -0700458 goto out;
459 }
Yang Shid8835442019-08-13 15:37:15 -0700460 if (!queue_pages_required(page, qp))
Naoya Horiguchic8633792017-09-08 16:11:08 -0700461 goto unlock;
Naoya Horiguchic8633792017-09-08 16:11:08 -0700462
Naoya Horiguchic8633792017-09-08 16:11:08 -0700463 flags = qp->flags;
464 /* go to thp migration */
Yang Shia7f40cf2019-03-28 20:43:55 -0700465 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
466 if (!vma_migratable(walk->vma)) {
Yang Shid8835442019-08-13 15:37:15 -0700467 ret = 1;
Yang Shia7f40cf2019-03-28 20:43:55 -0700468 goto unlock;
469 }
470
Naoya Horiguchic8633792017-09-08 16:11:08 -0700471 migrate_page_add(page, qp->pagelist, flags);
Yang Shia7f40cf2019-03-28 20:43:55 -0700472 } else
473 ret = -EIO;
Naoya Horiguchic8633792017-09-08 16:11:08 -0700474unlock:
475 spin_unlock(ptl);
476out:
477 return ret;
478}
479
Naoya Horiguchi88aaa2a2017-09-08 16:10:42 -0700480/*
Naoya Horiguchi98094942013-09-11 14:22:14 -0700481 * Scan through pages checking if pages follow certain conditions,
482 * and move them to the pagelist if they do.
Yang Shid8835442019-08-13 15:37:15 -0700483 *
484 * queue_pages_pte_range() has three possible return values:
485 * 0 - pages are placed on the right node or queued successfully.
486 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
487 * specified.
488 * -EIO - only MPOL_MF_STRICT was specified and an existing page was already
489 * on a node that does not follow the policy.
Naoya Horiguchi98094942013-09-11 14:22:14 -0700490 */
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800491static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
492 unsigned long end, struct mm_walk *walk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493{
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800494 struct vm_area_struct *vma = walk->vma;
495 struct page *page;
496 struct queue_pages *qp = walk->private;
497 unsigned long flags = qp->flags;
Naoya Horiguchic8633792017-09-08 16:11:08 -0700498 int ret;
Yang Shid8835442019-08-13 15:37:15 -0700499 bool has_unmovable = false;
Hugh Dickins91612e02005-06-21 17:15:07 -0700500 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700501 spinlock_t *ptl;
Hugh Dickins941150a2005-06-21 17:15:06 -0700502
Naoya Horiguchic8633792017-09-08 16:11:08 -0700503 ptl = pmd_trans_huge_lock(pmd, vma);
504 if (ptl) {
505 ret = queue_pages_pmd(pmd, ptl, addr, end, walk);
Yang Shid8835442019-08-13 15:37:15 -0700506 if (ret != 2)
Yang Shia7f40cf2019-03-28 20:43:55 -0700507 return ret;
Kirill A. Shutemov248db922016-01-15 16:54:14 -0800508 }
Yang Shid8835442019-08-13 15:37:15 -0700509 /* THP was split, fall through to pte walk */
Hugh Dickins91612e02005-06-21 17:15:07 -0700510
Naoya Horiguchi337d9ab2016-07-26 15:24:03 -0700511 if (pmd_trans_unstable(pmd))
512 return 0;
Michal Hocko94723aa2018-04-10 16:30:07 -0700513
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800514 pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
515 for (; addr != end; pte++, addr += PAGE_SIZE) {
Hugh Dickins91612e02005-06-21 17:15:07 -0700516 if (!pte_present(*pte))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 continue;
Linus Torvalds6aab3412005-11-28 14:34:23 -0800518 page = vm_normal_page(vma, addr, *pte);
519 if (!page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 continue;
Nick Piggin053837f2006-01-18 17:42:27 -0800521 /*
Hugh Dickins62b61f62009-12-14 17:59:33 -0800522 * vm_normal_page() filters out zero pages, but there might
523 * still be PageReserved pages to skip, perhaps in a VDSO.
Nick Piggin053837f2006-01-18 17:42:27 -0800524 */
Hugh Dickinsb79bc0a2013-02-22 16:35:13 -0800525 if (PageReserved(page))
Christoph Lameterf4598c82006-01-12 01:05:20 -0800526 continue;
Naoya Horiguchi88aaa2a2017-09-08 16:10:42 -0700527 if (!queue_pages_required(page, qp))
Christoph Lameter38e35862006-01-08 01:01:01 -0800528 continue;
Yang Shia7f40cf2019-03-28 20:43:55 -0700529 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
Yang Shid8835442019-08-13 15:37:15 -0700530 /* MPOL_MF_STRICT must be specified if we get here */
531 if (!vma_migratable(vma)) {
532 has_unmovable = true;
Yang Shia7f40cf2019-03-28 20:43:55 -0700533 break;
Yang Shid8835442019-08-13 15:37:15 -0700534 }
Yang Shia7f40cf2019-03-28 20:43:55 -0700535 migrate_page_add(page, qp->pagelist, flags);
536 } else
537 break;
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800538 }
539 pte_unmap_unlock(pte - 1, ptl);
540 cond_resched();
Yang Shid8835442019-08-13 15:37:15 -0700541
542 if (has_unmovable)
543 return 1;
544
Yang Shia7f40cf2019-03-28 20:43:55 -0700545 return addr != end ? -EIO : 0;
Hugh Dickins91612e02005-06-21 17:15:07 -0700546}
547
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800548static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask,
549 unsigned long addr, unsigned long end,
550 struct mm_walk *walk)
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700551{
552#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800553 struct queue_pages *qp = walk->private;
554 unsigned long flags = qp->flags;
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700555 struct page *page;
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800556 spinlock_t *ptl;
Naoya Horiguchid4c54912014-06-06 10:00:01 -0400557 pte_t entry;
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700558
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800559 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
560 entry = huge_ptep_get(pte);
Naoya Horiguchid4c54912014-06-06 10:00:01 -0400561 if (!pte_present(entry))
562 goto unlock;
563 page = pte_page(entry);
Naoya Horiguchi88aaa2a2017-09-08 16:10:42 -0700564 if (!queue_pages_required(page, qp))
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700565 goto unlock;
566 /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
567 if (flags & (MPOL_MF_MOVE_ALL) ||
568 (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800569 isolate_huge_page(page, qp->pagelist);
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700570unlock:
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800571 spin_unlock(ptl);
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700572#else
573 BUG();
574#endif
Hugh Dickins91612e02005-06-21 17:15:07 -0700575 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576}
577
Aneesh Kumar K.V58772312013-12-06 00:08:22 +0530578#ifdef CONFIG_NUMA_BALANCING
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200579/*
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200580 * This is used to mark a range of virtual addresses to be inaccessible.
581 * These are later cleared by a NUMA hinting fault. Depending on these
582 * faults, pages may be migrated for better NUMA placement.
583 *
584 * This is assuming that NUMA faults are handled using PROT_NONE. If
585 * an architecture makes a different choice, it will need further
586 * changes to the core.
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200587 */
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200588unsigned long change_prot_numa(struct vm_area_struct *vma,
589 unsigned long addr, unsigned long end)
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200590{
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200591 int nr_updated;
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200592
Mel Gorman4d942462015-02-12 14:58:28 -0800593 nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1);
Mel Gorman03c5a6e2012-11-02 14:52:48 +0000594 if (nr_updated)
595 count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200596
Mel Gorman4b10e7d2012-10-25 14:16:32 +0200597 return nr_updated;
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200598}
599#else
600static unsigned long change_prot_numa(struct vm_area_struct *vma,
601 unsigned long addr, unsigned long end)
602{
603 return 0;
604}
Aneesh Kumar K.V58772312013-12-06 00:08:22 +0530605#endif /* CONFIG_NUMA_BALANCING */
Lee Schermerhornb24f53a2012-10-25 14:16:32 +0200606
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800607static int queue_pages_test_walk(unsigned long start, unsigned long end,
608 struct mm_walk *walk)
609{
610 struct vm_area_struct *vma = walk->vma;
611 struct queue_pages *qp = walk->private;
612 unsigned long endvma = vma->vm_end;
613 unsigned long flags = qp->flags;
614
Yang Shia7f40cf2019-03-28 20:43:55 -0700615 /*
616 * Need check MPOL_MF_STRICT to return -EIO if possible
617 * regardless of vma_migratable
618 */
619 if (!vma_migratable(vma) &&
620 !(flags & MPOL_MF_STRICT))
Naoya Horiguchi48684a62015-02-11 15:28:06 -0800621 return 1;
622
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800623 if (endvma > end)
624 endvma = end;
625 if (vma->vm_start > start)
626 start = vma->vm_start;
627
628 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
629 if (!vma->vm_next && vma->vm_end < end)
630 return -EFAULT;
631 if (qp->prev && qp->prev->vm_end < vma->vm_start)
632 return -EFAULT;
633 }
634
635 qp->prev = vma;
636
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800637 if (flags & MPOL_MF_LAZY) {
638 /* Similar to task_numa_work, skip inaccessible VMAs */
Liang Chen4355c012016-03-15 14:56:42 -0700639 if (!is_vm_hugetlb_page(vma) &&
640 (vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)) &&
641 !(vma->vm_flags & VM_MIXEDMAP))
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800642 change_prot_numa(vma, start, endvma);
643 return 1;
644 }
645
Kirill A. Shutemov77bf45e2016-02-05 15:36:33 -0800646 /* queue pages from current vma */
Yang Shia7f40cf2019-03-28 20:43:55 -0700647 if (flags & MPOL_MF_VALID)
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800648 return 0;
649 return 1;
650}
651
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800652/*
Naoya Horiguchi98094942013-09-11 14:22:14 -0700653 * Walk through page tables and collect pages to be migrated.
654 *
655 * If pages found in a given range are on a set of nodes (determined by
656 * @nodes and @flags,) it's isolated and queued to the pagelist which is
Yang Shid8835442019-08-13 15:37:15 -0700657 * passed via @private.
658 *
659 * queue_pages_range() has three possible return values:
660 * 1 - there is unmovable page, but MPOL_MF_MOVE* & MPOL_MF_STRICT were
661 * specified.
662 * 0 - queue pages successfully or no misplaced page.
663 * -EIO - there is misplaced page and only MPOL_MF_STRICT was specified.
Christoph Lameterdc9aa5b2006-01-08 01:00:50 -0800664 */
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -0700665static int
Naoya Horiguchi98094942013-09-11 14:22:14 -0700666queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800667 nodemask_t *nodes, unsigned long flags,
668 struct list_head *pagelist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669{
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800670 struct queue_pages qp = {
671 .pagelist = pagelist,
672 .flags = flags,
673 .nmask = nodes,
674 .prev = NULL,
675 };
676 struct mm_walk queue_pages_walk = {
677 .hugetlb_entry = queue_pages_hugetlb,
678 .pmd_entry = queue_pages_pte_range,
679 .test_walk = queue_pages_test_walk,
680 .mm = mm,
681 .private = &qp,
682 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
Naoya Horiguchi6f4576e2015-02-11 15:28:03 -0800684 return walk_page_range(start, end, &queue_pages_walk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685}
686
KOSAKI Motohiro869833f2012-10-08 16:29:16 -0700687/*
688 * Apply policy to a single VMA
689 * This must be called with the mmap_sem held for writing.
690 */
691static int vma_replace_policy(struct vm_area_struct *vma,
692 struct mempolicy *pol)
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700693{
KOSAKI Motohiro869833f2012-10-08 16:29:16 -0700694 int err;
695 struct mempolicy *old;
696 struct mempolicy *new;
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700697
698 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
699 vma->vm_start, vma->vm_end, vma->vm_pgoff,
700 vma->vm_ops, vma->vm_file,
701 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
702
KOSAKI Motohiro869833f2012-10-08 16:29:16 -0700703 new = mpol_dup(pol);
704 if (IS_ERR(new))
705 return PTR_ERR(new);
706
707 if (vma->vm_ops && vma->vm_ops->set_policy) {
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700708 err = vma->vm_ops->set_policy(vma, new);
KOSAKI Motohiro869833f2012-10-08 16:29:16 -0700709 if (err)
710 goto err_out;
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700711 }
KOSAKI Motohiro869833f2012-10-08 16:29:16 -0700712
713 old = vma->vm_policy;
714 vma->vm_policy = new; /* protected by mmap_sem */
715 mpol_put(old);
716
717 return 0;
718 err_out:
719 mpol_put(new);
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700720 return err;
721}
722
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723/* Step 2: apply policy to a range and do splits. */
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800724static int mbind_range(struct mm_struct *mm, unsigned long start,
725 unsigned long end, struct mempolicy *new_pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726{
727 struct vm_area_struct *next;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800728 struct vm_area_struct *prev;
729 struct vm_area_struct *vma;
730 int err = 0;
KOSAKI Motohiroe26a5112011-12-28 15:57:11 -0800731 pgoff_t pgoff;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800732 unsigned long vmstart;
733 unsigned long vmend;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734
Linus Torvalds097d5912012-03-06 18:23:36 -0800735 vma = find_vma(mm, start);
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800736 if (!vma || vma->vm_start > start)
737 return -EFAULT;
738
Linus Torvalds097d5912012-03-06 18:23:36 -0800739 prev = vma->vm_prev;
KOSAKI Motohiroe26a5112011-12-28 15:57:11 -0800740 if (start > vma->vm_start)
741 prev = vma;
742
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800743 for (; vma && vma->vm_start < end; prev = vma, vma = next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 next = vma->vm_next;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800745 vmstart = max(start, vma->vm_start);
746 vmend = min(end, vma->vm_end);
747
KOSAKI Motohiroe26a5112011-12-28 15:57:11 -0800748 if (mpol_equal(vma_policy(vma), new_pol))
749 continue;
750
751 pgoff = vma->vm_pgoff +
752 ((vmstart - vma->vm_start) >> PAGE_SHIFT);
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800753 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
Andrea Arcangeli19a809a2015-09-04 15:46:24 -0700754 vma->anon_vma, vma->vm_file, pgoff,
755 new_pol, vma->vm_userfaultfd_ctx);
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800756 if (prev) {
757 vma = prev;
758 next = vma->vm_next;
Oleg Nesterov3964acd2013-07-31 13:53:28 -0700759 if (mpol_equal(vma_policy(vma), new_pol))
760 continue;
761 /* vma_merge() joined vma && vma->next, case 8 */
762 goto replace;
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800763 }
764 if (vma->vm_start != vmstart) {
765 err = split_vma(vma->vm_mm, vma, vmstart, 1);
766 if (err)
767 goto out;
768 }
769 if (vma->vm_end != vmend) {
770 err = split_vma(vma->vm_mm, vma, vmend, 0);
771 if (err)
772 goto out;
773 }
Oleg Nesterov3964acd2013-07-31 13:53:28 -0700774 replace:
KOSAKI Motohiro869833f2012-10-08 16:29:16 -0700775 err = vma_replace_policy(vma, new_pol);
KOSAKI Motohiro8d34694c12012-10-08 16:29:14 -0700776 if (err)
777 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 }
KOSAKI Motohiro9d8cebd2010-03-05 13:41:57 -0800779
780 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 return err;
782}
783
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784/* Set the process memory policy */
David Rientjes028fec42008-04-28 02:12:25 -0700785static long do_set_mempolicy(unsigned short mode, unsigned short flags,
786 nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787{
Miao Xie58568d22009-06-16 15:31:49 -0700788 struct mempolicy *new, *old;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700789 NODEMASK_SCRATCH(scratch);
Miao Xie58568d22009-06-16 15:31:49 -0700790 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700792 if (!scratch)
793 return -ENOMEM;
Lee Schermerhornf4e53d92008-04-28 02:13:10 -0700794
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700795 new = mpol_new(mode, flags, nodes);
796 if (IS_ERR(new)) {
797 ret = PTR_ERR(new);
798 goto out;
799 }
Oleg Nesterov2c7c3a72014-10-09 15:27:55 -0700800
Miao Xie58568d22009-06-16 15:31:49 -0700801 task_lock(current);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700802 ret = mpol_set_nodemask(new, nodes, scratch);
Miao Xie58568d22009-06-16 15:31:49 -0700803 if (ret) {
804 task_unlock(current);
Miao Xie58568d22009-06-16 15:31:49 -0700805 mpol_put(new);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700806 goto out;
Miao Xie58568d22009-06-16 15:31:49 -0700807 }
808 old = current->mempolicy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 current->mempolicy = new;
Vlastimil Babka45816682017-07-06 15:39:59 -0700810 if (new && new->mode == MPOL_INTERLEAVE)
811 current->il_prev = MAX_NUMNODES-1;
Miao Xie58568d22009-06-16 15:31:49 -0700812 task_unlock(current);
Miao Xie58568d22009-06-16 15:31:49 -0700813 mpol_put(old);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -0700814 ret = 0;
815out:
816 NODEMASK_SCRATCH_FREE(scratch);
817 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818}
819
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700820/*
821 * Return nodemask for policy for get_mempolicy() query
Miao Xie58568d22009-06-16 15:31:49 -0700822 *
823 * Called with task's alloc_lock held
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700824 */
825static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826{
Andi Kleendfcd3c02005-10-29 18:15:48 -0700827 nodes_clear(*nodes);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700828 if (p == &default_policy)
829 return;
830
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700831 switch (p->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -0700832 case MPOL_BIND:
833 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 case MPOL_INTERLEAVE:
Andi Kleendfcd3c02005-10-29 18:15:48 -0700835 *nodes = p->v.nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 break;
837 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -0700838 if (!(p->flags & MPOL_F_LOCAL))
Andi Kleendfcd3c02005-10-29 18:15:48 -0700839 node_set(p->v.preferred_node, *nodes);
Lee Schermerhorn53f25562008-04-28 02:13:20 -0700840 /* else return empty node mask for local allocation */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 break;
842 default:
843 BUG();
844 }
845}
846
Andrea Arcangeli3b9aadf2018-10-26 15:05:16 -0700847static int lookup_node(struct mm_struct *mm, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848{
849 struct page *p;
850 int err;
851
Andrea Arcangeli3b9aadf2018-10-26 15:05:16 -0700852 int locked = 1;
853 err = get_user_pages_locked(addr & PAGE_MASK, 1, 0, &p, &locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854 if (err >= 0) {
855 err = page_to_nid(p);
856 put_page(p);
857 }
Andrea Arcangeli3b9aadf2018-10-26 15:05:16 -0700858 if (locked)
859 up_read(&mm->mmap_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 return err;
861}
862
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863/* Retrieve NUMA policy */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -0700864static long do_get_mempolicy(int *policy, nodemask_t *nmask,
865 unsigned long addr, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866{
Christoph Lameter8bccd852005-10-29 18:16:59 -0700867 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 struct mm_struct *mm = current->mm;
869 struct vm_area_struct *vma = NULL;
Andrea Arcangeli3b9aadf2018-10-26 15:05:16 -0700870 struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700872 if (flags &
873 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 return -EINVAL;
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700875
876 if (flags & MPOL_F_MEMS_ALLOWED) {
877 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
878 return -EINVAL;
879 *policy = 0; /* just so it's initialized */
Miao Xie58568d22009-06-16 15:31:49 -0700880 task_lock(current);
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700881 *nmask = cpuset_current_mems_allowed;
Miao Xie58568d22009-06-16 15:31:49 -0700882 task_unlock(current);
Lee Schermerhorn754af6f2007-10-16 01:24:51 -0700883 return 0;
884 }
885
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 if (flags & MPOL_F_ADDR) {
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700887 /*
888 * Do NOT fall back to task policy if the
889 * vma/shared policy at addr is NULL. We
890 * want to return MPOL_DEFAULT in this case.
891 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 down_read(&mm->mmap_sem);
893 vma = find_vma_intersection(mm, addr, addr+1);
894 if (!vma) {
895 up_read(&mm->mmap_sem);
896 return -EFAULT;
897 }
898 if (vma->vm_ops && vma->vm_ops->get_policy)
899 pol = vma->vm_ops->get_policy(vma, addr);
900 else
901 pol = vma->vm_policy;
902 } else if (addr)
903 return -EINVAL;
904
905 if (!pol)
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700906 pol = &default_policy; /* indicates default behavior */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907
908 if (flags & MPOL_F_NODE) {
909 if (flags & MPOL_F_ADDR) {
Andrea Arcangeli3b9aadf2018-10-26 15:05:16 -0700910 /*
911 * Take a refcount on the mpol, lookup_node()
912 * wil drop the mmap_sem, so after calling
913 * lookup_node() only "pol" remains valid, "vma"
914 * is stale.
915 */
916 pol_refcount = pol;
917 vma = NULL;
918 mpol_get(pol);
919 err = lookup_node(mm, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 if (err < 0)
921 goto out;
Christoph Lameter8bccd852005-10-29 18:16:59 -0700922 *policy = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 } else if (pol == current->mempolicy &&
Lee Schermerhorn45c47452008-04-28 02:13:12 -0700924 pol->mode == MPOL_INTERLEAVE) {
Vlastimil Babka45816682017-07-06 15:39:59 -0700925 *policy = next_node_in(current->il_prev, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 } else {
927 err = -EINVAL;
928 goto out;
929 }
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700930 } else {
931 *policy = pol == &default_policy ? MPOL_DEFAULT :
932 pol->mode;
David Rientjesd79df632008-07-04 12:24:13 -0700933 /*
934 * Internal mempolicy flags must be masked off before exposing
935 * the policy to userspace.
936 */
937 *policy |= (pol->flags & MPOL_MODE_FLAGS);
Lee Schermerhornbea904d2008-04-28 02:13:18 -0700938 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 err = 0;
Miao Xie58568d22009-06-16 15:31:49 -0700941 if (nmask) {
Lee Schermerhornc6b6ef82010-03-23 13:35:41 -0700942 if (mpol_store_user_nodemask(pol)) {
943 *nmask = pol->w.user_nodemask;
944 } else {
945 task_lock(current);
946 get_policy_nodemask(pol, nmask);
947 task_unlock(current);
948 }
Miao Xie58568d22009-06-16 15:31:49 -0700949 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950
951 out:
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700952 mpol_cond_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953 if (vma)
Andrea Arcangeli3b9aadf2018-10-26 15:05:16 -0700954 up_read(&mm->mmap_sem);
955 if (pol_refcount)
956 mpol_put(pol_refcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 return err;
958}
959
Christoph Lameterb20a3502006-03-22 00:09:12 -0800960#ifdef CONFIG_MIGRATION
Christoph Lameter8bccd852005-10-29 18:16:59 -0700961/*
Naoya Horiguchic8633792017-09-08 16:11:08 -0700962 * page migration, thp tail pages can be passed.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800963 */
Christoph Lameterfc301282006-01-18 17:42:29 -0800964static void migrate_page_add(struct page *page, struct list_head *pagelist,
965 unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800966{
Naoya Horiguchic8633792017-09-08 16:11:08 -0700967 struct page *head = compound_head(page);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800968 /*
Christoph Lameterfc301282006-01-18 17:42:29 -0800969 * Avoid migrating a page that is shared with others.
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800970 */
Naoya Horiguchic8633792017-09-08 16:11:08 -0700971 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(head) == 1) {
972 if (!isolate_lru_page(head)) {
973 list_add_tail(&head->lru, pagelist);
974 mod_node_page_state(page_pgdat(head),
975 NR_ISOLATED_ANON + page_is_file_cache(head),
976 hpage_nr_pages(head));
Nick Piggin62695a82008-10-18 20:26:09 -0700977 }
978 }
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -0800979}
980
Michal Hockoa49bd4d2018-04-10 16:29:59 -0700981/* page allocation callback for NUMA node migration */
Michal Hocko666feb22018-04-10 16:30:03 -0700982struct page *alloc_new_node_page(struct page *page, unsigned long node)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700983{
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -0700984 if (PageHuge(page))
985 return alloc_huge_page_node(page_hstate(compound_head(page)),
986 node);
Michal Hocko94723aa2018-04-10 16:30:07 -0700987 else if (PageTransHuge(page)) {
Naoya Horiguchic8633792017-09-08 16:11:08 -0700988 struct page *thp;
989
990 thp = alloc_pages_node(node,
991 (GFP_TRANSHUGE | __GFP_THISNODE),
992 HPAGE_PMD_ORDER);
993 if (!thp)
994 return NULL;
995 prep_transhuge_page(thp);
996 return thp;
997 } else
Vlastimil Babka96db8002015-09-08 15:03:50 -0700998 return __alloc_pages_node(node, GFP_HIGHUSER_MOVABLE |
David Rientjesb360edb2015-04-14 15:46:52 -0700999 __GFP_THISNODE, 0);
Christoph Lameter95a402c2006-06-23 02:03:53 -07001000}
1001
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001002/*
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001003 * Migrate pages from one node to a target node.
1004 * Returns error or the number of pages not migrated.
1005 */
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001006static int migrate_to_node(struct mm_struct *mm, int source, int dest,
1007 int flags)
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001008{
1009 nodemask_t nmask;
1010 LIST_HEAD(pagelist);
1011 int err = 0;
1012
1013 nodes_clear(nmask);
1014 node_set(source, nmask);
1015
Minchan Kim08270802012-10-08 16:33:38 -07001016 /*
1017 * This does not "check" the range but isolates all pages that
1018 * need migration. Between passing in the full user address
1019 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
1020 */
1021 VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
Naoya Horiguchi98094942013-09-11 14:22:14 -07001022 queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001023 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1024
Minchan Kimcf608ac2010-10-26 14:21:29 -07001025 if (!list_empty(&pagelist)) {
Michal Hockoa49bd4d2018-04-10 16:29:59 -07001026 err = migrate_pages(&pagelist, alloc_new_node_page, NULL, dest,
Hugh Dickins9c620e22013-02-22 16:35:14 -08001027 MIGRATE_SYNC, MR_SYSCALL);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001028 if (err)
Naoya Horiguchie2d8cf42013-09-11 14:22:03 -07001029 putback_movable_pages(&pagelist);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001030 }
Christoph Lameter95a402c2006-06-23 02:03:53 -07001031
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001032 return err;
1033}
1034
1035/*
1036 * Move pages between the two nodesets so as to preserve the physical
1037 * layout as much as possible.
Christoph Lameter39743882006-01-08 01:00:51 -08001038 *
1039 * Returns the number of page that could not be moved.
1040 */
Andrew Morton0ce72d42012-05-29 15:06:24 -07001041int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1042 const nodemask_t *to, int flags)
Christoph Lameter39743882006-01-08 01:00:51 -08001043{
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001044 int busy = 0;
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001045 int err;
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001046 nodemask_t tmp;
Christoph Lameter39743882006-01-08 01:00:51 -08001047
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001048 err = migrate_prep();
1049 if (err)
1050 return err;
1051
Lee Schermerhorn53f25562008-04-28 02:13:20 -07001052 down_read(&mm->mmap_sem);
Christoph Lameter39743882006-01-08 01:00:51 -08001053
KOSAKI Motohiroda0aa132010-03-05 13:41:59 -08001054 /*
1055 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1056 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
1057 * bit in 'tmp', and return that <source, dest> pair for migration.
1058 * The pair of nodemasks 'to' and 'from' define the map.
1059 *
1060 * If no pair of bits is found that way, fallback to picking some
1061 * pair of 'source' and 'dest' bits that are not the same. If the
1062 * 'source' and 'dest' bits are the same, this represents a node
1063 * that will be migrating to itself, so no pages need move.
1064 *
1065 * If no bits are left in 'tmp', or if all remaining bits left
1066 * in 'tmp' correspond to the same bit in 'to', return false
1067 * (nothing left to migrate).
1068 *
1069 * This lets us pick a pair of nodes to migrate between, such that
1070 * if possible the dest node is not already occupied by some other
1071 * source node, minimizing the risk of overloading the memory on a
1072 * node that would happen if we migrated incoming memory to a node
1073 * before migrating outgoing memory source that same node.
1074 *
1075 * A single scan of tmp is sufficient. As we go, we remember the
1076 * most recent <s, d> pair that moved (s != d). If we find a pair
1077 * that not only moved, but what's better, moved to an empty slot
1078 * (d is not set in tmp), then we break out then, with that pair.
Justin P. Mattockae0e47f2011-03-01 15:06:02 +01001079 * Otherwise when we finish scanning from_tmp, we at least have the
KOSAKI Motohiroda0aa132010-03-05 13:41:59 -08001080 * most recent <s, d> pair that moved. If we get all the way through
1081 * the scan of tmp without finding any node that moved, much less
1082 * moved to an empty node, then there is nothing left worth migrating.
1083 */
Christoph Lameterd4984712006-01-08 01:00:55 -08001084
Andrew Morton0ce72d42012-05-29 15:06:24 -07001085 tmp = *from;
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001086 while (!nodes_empty(tmp)) {
1087 int s,d;
Jianguo Wub76ac7e2013-11-12 15:07:39 -08001088 int source = NUMA_NO_NODE;
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001089 int dest = 0;
1090
1091 for_each_node_mask(s, tmp) {
Larry Woodman4a5b18c2012-05-29 15:06:24 -07001092
1093 /*
1094 * do_migrate_pages() tries to maintain the relative
1095 * node relationship of the pages established between
1096 * threads and memory areas.
1097 *
1098 * However if the number of source nodes is not equal to
1099 * the number of destination nodes we can not preserve
1100 * this node relative relationship. In that case, skip
1101 * copying memory from a node that is in the destination
1102 * mask.
1103 *
1104 * Example: [2,3,4] -> [3,4,5] moves everything.
1105 * [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1106 */
1107
Andrew Morton0ce72d42012-05-29 15:06:24 -07001108 if ((nodes_weight(*from) != nodes_weight(*to)) &&
1109 (node_isset(s, *to)))
Larry Woodman4a5b18c2012-05-29 15:06:24 -07001110 continue;
1111
Andrew Morton0ce72d42012-05-29 15:06:24 -07001112 d = node_remap(s, *from, *to);
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001113 if (s == d)
1114 continue;
1115
1116 source = s; /* Node moved. Memorize */
1117 dest = d;
1118
1119 /* dest not in remaining from nodes? */
1120 if (!node_isset(dest, tmp))
1121 break;
1122 }
Jianguo Wub76ac7e2013-11-12 15:07:39 -08001123 if (source == NUMA_NO_NODE)
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001124 break;
1125
1126 node_clear(source, tmp);
1127 err = migrate_to_node(mm, source, dest, flags);
1128 if (err > 0)
1129 busy += err;
1130 if (err < 0)
1131 break;
Christoph Lameter39743882006-01-08 01:00:51 -08001132 }
1133 up_read(&mm->mmap_sem);
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001134 if (err < 0)
1135 return err;
1136 return busy;
Christoph Lameterb20a3502006-03-22 00:09:12 -08001137
Christoph Lameter39743882006-01-08 01:00:51 -08001138}
1139
Lee Schermerhorn3ad33b242007-11-14 16:59:10 -08001140/*
1141 * Allocate a new page for page migration based on vma policy.
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001142 * Start by assuming the page is mapped by the same vma as contains @start.
Lee Schermerhorn3ad33b242007-11-14 16:59:10 -08001143 * Search forward from there, if not. N.B., this assumes that the
1144 * list of pages handed to migrate_pages()--which is how we get here--
1145 * is in virtual address order.
1146 */
Michal Hocko666feb22018-04-10 16:30:03 -07001147static struct page *new_page(struct page *page, unsigned long start)
Christoph Lameter95a402c2006-06-23 02:03:53 -07001148{
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001149 struct vm_area_struct *vma;
Lee Schermerhorn3ad33b242007-11-14 16:59:10 -08001150 unsigned long uninitialized_var(address);
Christoph Lameter95a402c2006-06-23 02:03:53 -07001151
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001152 vma = find_vma(current->mm, start);
Lee Schermerhorn3ad33b242007-11-14 16:59:10 -08001153 while (vma) {
1154 address = page_address_in_vma(page, vma);
1155 if (address != -EFAULT)
1156 break;
1157 vma = vma->vm_next;
1158 }
1159
Wanpeng Li11c731e2013-12-18 17:08:56 -08001160 if (PageHuge(page)) {
Michal Hocko389c8172018-01-31 16:21:03 -08001161 return alloc_huge_page_vma(page_hstate(compound_head(page)),
1162 vma, address);
Michal Hocko94723aa2018-04-10 16:30:07 -07001163 } else if (PageTransHuge(page)) {
Naoya Horiguchic8633792017-09-08 16:11:08 -07001164 struct page *thp;
1165
David Rientjes356ff8a2018-12-07 14:50:16 -08001166 thp = alloc_hugepage_vma(GFP_TRANSHUGE, vma, address,
1167 HPAGE_PMD_ORDER);
Naoya Horiguchic8633792017-09-08 16:11:08 -07001168 if (!thp)
1169 return NULL;
1170 prep_transhuge_page(thp);
1171 return thp;
Wanpeng Li11c731e2013-12-18 17:08:56 -08001172 }
1173 /*
1174 * if !vma, alloc_page_vma() will use task or system default policy
1175 */
Michal Hocko0f556852017-07-12 14:36:58 -07001176 return alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL,
1177 vma, address);
Christoph Lameter95a402c2006-06-23 02:03:53 -07001178}
Christoph Lameterb20a3502006-03-22 00:09:12 -08001179#else
1180
1181static void migrate_page_add(struct page *page, struct list_head *pagelist,
1182 unsigned long flags)
1183{
1184}
1185
Andrew Morton0ce72d42012-05-29 15:06:24 -07001186int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1187 const nodemask_t *to, int flags)
Christoph Lameterb20a3502006-03-22 00:09:12 -08001188{
1189 return -ENOSYS;
1190}
Christoph Lameter95a402c2006-06-23 02:03:53 -07001191
Michal Hocko666feb22018-04-10 16:30:03 -07001192static struct page *new_page(struct page *page, unsigned long start)
Christoph Lameter95a402c2006-06-23 02:03:53 -07001193{
1194 return NULL;
1195}
Christoph Lameterb20a3502006-03-22 00:09:12 -08001196#endif
1197
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001198static long do_mbind(unsigned long start, unsigned long len,
David Rientjes028fec42008-04-28 02:12:25 -07001199 unsigned short mode, unsigned short mode_flags,
1200 nodemask_t *nmask, unsigned long flags)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001201{
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001202 struct mm_struct *mm = current->mm;
1203 struct mempolicy *new;
1204 unsigned long end;
1205 int err;
Yang Shid8835442019-08-13 15:37:15 -07001206 int ret;
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001207 LIST_HEAD(pagelist);
1208
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001209 if (flags & ~(unsigned long)MPOL_MF_VALID)
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001210 return -EINVAL;
Christoph Lameter74c00242006-03-14 19:50:21 -08001211 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001212 return -EPERM;
1213
1214 if (start & ~PAGE_MASK)
1215 return -EINVAL;
1216
1217 if (mode == MPOL_DEFAULT)
1218 flags &= ~MPOL_MF_STRICT;
1219
1220 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1221 end = start + len;
1222
1223 if (end < start)
1224 return -EINVAL;
1225 if (end == start)
1226 return 0;
1227
David Rientjes028fec42008-04-28 02:12:25 -07001228 new = mpol_new(mode, mode_flags, nmask);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001229 if (IS_ERR(new))
1230 return PTR_ERR(new);
1231
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001232 if (flags & MPOL_MF_LAZY)
1233 new->flags |= MPOL_F_MOF;
1234
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001235 /*
1236 * If we are using the default policy then operation
1237 * on discontinuous address spaces is okay after all
1238 */
1239 if (!new)
1240 flags |= MPOL_MF_DISCONTIG_OK;
1241
David Rientjes028fec42008-04-28 02:12:25 -07001242 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1243 start, start + len, mode, mode_flags,
David Rientjes00ef2d22013-02-22 16:35:36 -08001244 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001245
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001246 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1247
1248 err = migrate_prep();
1249 if (err)
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001250 goto mpol_out;
Christoph Lameter0aedadf2008-11-06 12:53:30 -08001251 }
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07001252 {
1253 NODEMASK_SCRATCH(scratch);
1254 if (scratch) {
1255 down_write(&mm->mmap_sem);
1256 task_lock(current);
1257 err = mpol_set_nodemask(new, nmask, scratch);
1258 task_unlock(current);
1259 if (err)
1260 up_write(&mm->mmap_sem);
1261 } else
1262 err = -ENOMEM;
1263 NODEMASK_SCRATCH_FREE(scratch);
1264 }
KOSAKI Motohirob05ca732009-10-26 16:49:59 -07001265 if (err)
1266 goto mpol_out;
1267
Yang Shid8835442019-08-13 15:37:15 -07001268 ret = queue_pages_range(mm, start, end, nmask,
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001269 flags | MPOL_MF_INVERT, &pagelist);
Yang Shid8835442019-08-13 15:37:15 -07001270
1271 if (ret < 0) {
1272 err = -EIO;
1273 goto up_out;
1274 }
1275
1276 err = mbind_range(mm, start, end, new);
Christoph Lameter7e2ab152006-02-01 03:05:40 -08001277
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001278 if (!err) {
1279 int nr_failed = 0;
1280
Minchan Kimcf608ac2010-10-26 14:21:29 -07001281 if (!list_empty(&pagelist)) {
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001282 WARN_ON_ONCE(flags & MPOL_MF_LAZY);
Hugh Dickinsd05f0cdcb2014-06-23 13:22:07 -07001283 nr_failed = migrate_pages(&pagelist, new_page, NULL,
1284 start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001285 if (nr_failed)
Naoya Horiguchi74060e42013-09-11 14:22:06 -07001286 putback_movable_pages(&pagelist);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001287 }
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001288
Yang Shid8835442019-08-13 15:37:15 -07001289 if ((ret > 0) || (nr_failed && (flags & MPOL_MF_STRICT)))
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001290 err = -EIO;
KOSAKI Motohiroab8a3e12009-10-26 16:49:58 -07001291 } else
Joonsoo Kimb0e5fd72013-12-18 17:08:51 -08001292 putback_movable_pages(&pagelist);
Christoph Lameterb20a3502006-03-22 00:09:12 -08001293
Yang Shid8835442019-08-13 15:37:15 -07001294up_out:
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001295 up_write(&mm->mmap_sem);
Yang Shid8835442019-08-13 15:37:15 -07001296mpol_out:
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001297 mpol_put(new);
Christoph Lameter6ce3c4c2006-01-08 01:01:04 -08001298 return err;
1299}
1300
Christoph Lameter39743882006-01-08 01:00:51 -08001301/*
Christoph Lameter8bccd852005-10-29 18:16:59 -07001302 * User space interface with variable sized bitmaps for nodelists.
1303 */
1304
1305/* Copy a node mask from user space. */
Christoph Lameter39743882006-01-08 01:00:51 -08001306static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
Christoph Lameter8bccd852005-10-29 18:16:59 -07001307 unsigned long maxnode)
1308{
1309 unsigned long k;
Yisheng Xie56521e72018-01-31 16:16:11 -08001310 unsigned long t;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001311 unsigned long nlongs;
1312 unsigned long endmask;
1313
1314 --maxnode;
1315 nodes_clear(*nodes);
1316 if (maxnode == 0 || !nmask)
1317 return 0;
Andi Kleena9c930b2006-02-20 18:27:59 -08001318 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
Chris Wright636f13c2006-02-17 13:59:36 -08001319 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001320
1321 nlongs = BITS_TO_LONGS(maxnode);
1322 if ((maxnode % BITS_PER_LONG) == 0)
1323 endmask = ~0UL;
1324 else
1325 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1326
Yisheng Xie56521e72018-01-31 16:16:11 -08001327 /*
1328 * When the user specified more nodes than supported just check
1329 * if the non supported part is all zero.
1330 *
1331 * If maxnode have more longs than MAX_NUMNODES, check
1332 * the bits in that area first. And then go through to
1333 * check the rest bits which equal or bigger than MAX_NUMNODES.
1334 * Otherwise, just check bits [MAX_NUMNODES, maxnode).
1335 */
Christoph Lameter8bccd852005-10-29 18:16:59 -07001336 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
Christoph Lameter8bccd852005-10-29 18:16:59 -07001337 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
Christoph Lameter8bccd852005-10-29 18:16:59 -07001338 if (get_user(t, nmask + k))
1339 return -EFAULT;
1340 if (k == nlongs - 1) {
1341 if (t & endmask)
1342 return -EINVAL;
1343 } else if (t)
1344 return -EINVAL;
1345 }
1346 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1347 endmask = ~0UL;
1348 }
1349
Yisheng Xie56521e72018-01-31 16:16:11 -08001350 if (maxnode > MAX_NUMNODES && MAX_NUMNODES % BITS_PER_LONG != 0) {
1351 unsigned long valid_mask = endmask;
1352
1353 valid_mask &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1);
1354 if (get_user(t, nmask + nlongs - 1))
1355 return -EFAULT;
1356 if (t & valid_mask)
1357 return -EINVAL;
1358 }
1359
Christoph Lameter8bccd852005-10-29 18:16:59 -07001360 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1361 return -EFAULT;
1362 nodes_addr(*nodes)[nlongs-1] &= endmask;
1363 return 0;
1364}
1365
1366/* Copy a kernel node mask to user space */
1367static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1368 nodemask_t *nodes)
1369{
1370 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
Ralph Campbell050c17f2019-02-20 22:18:58 -08001371 unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001372
1373 if (copy > nbytes) {
1374 if (copy > PAGE_SIZE)
1375 return -EINVAL;
1376 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1377 return -EFAULT;
1378 copy = nbytes;
1379 }
1380 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1381}
1382
Dominik Brodowskie7dc9ad62018-03-17 16:12:22 +01001383static long kernel_mbind(unsigned long start, unsigned long len,
1384 unsigned long mode, const unsigned long __user *nmask,
1385 unsigned long maxnode, unsigned int flags)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001386{
1387 nodemask_t nodes;
1388 int err;
David Rientjes028fec42008-04-28 02:12:25 -07001389 unsigned short mode_flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001390
David Rientjes028fec42008-04-28 02:12:25 -07001391 mode_flags = mode & MPOL_MODE_FLAGS;
1392 mode &= ~MPOL_MODE_FLAGS;
David Rientjesa3b51e02008-04-28 02:12:23 -07001393 if (mode >= MPOL_MAX)
1394 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001395 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1396 (mode_flags & MPOL_F_RELATIVE_NODES))
1397 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001398 err = get_nodes(&nodes, nmask, maxnode);
1399 if (err)
1400 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001401 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001402}
1403
Dominik Brodowskie7dc9ad62018-03-17 16:12:22 +01001404SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1405 unsigned long, mode, const unsigned long __user *, nmask,
1406 unsigned long, maxnode, unsigned int, flags)
1407{
1408 return kernel_mbind(start, len, mode, nmask, maxnode, flags);
1409}
1410
Christoph Lameter8bccd852005-10-29 18:16:59 -07001411/* Set the process memory policy */
Dominik Brodowskiaf03c4a2018-03-17 16:20:01 +01001412static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
1413 unsigned long maxnode)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001414{
1415 int err;
1416 nodemask_t nodes;
David Rientjes028fec42008-04-28 02:12:25 -07001417 unsigned short flags;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001418
David Rientjes028fec42008-04-28 02:12:25 -07001419 flags = mode & MPOL_MODE_FLAGS;
1420 mode &= ~MPOL_MODE_FLAGS;
1421 if ((unsigned int)mode >= MPOL_MAX)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001422 return -EINVAL;
David Rientjes4c50bc02008-04-28 02:12:30 -07001423 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1424 return -EINVAL;
Christoph Lameter8bccd852005-10-29 18:16:59 -07001425 err = get_nodes(&nodes, nmask, maxnode);
1426 if (err)
1427 return err;
David Rientjes028fec42008-04-28 02:12:25 -07001428 return do_set_mempolicy(mode, flags, &nodes);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001429}
1430
Dominik Brodowskiaf03c4a2018-03-17 16:20:01 +01001431SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
1432 unsigned long, maxnode)
1433{
1434 return kernel_set_mempolicy(mode, nmask, maxnode);
1435}
1436
Dominik Brodowskib6e9b0b2018-03-17 16:00:25 +01001437static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
1438 const unsigned long __user *old_nodes,
1439 const unsigned long __user *new_nodes)
Christoph Lameter39743882006-01-08 01:00:51 -08001440{
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001441 struct mm_struct *mm = NULL;
Christoph Lameter39743882006-01-08 01:00:51 -08001442 struct task_struct *task;
Christoph Lameter39743882006-01-08 01:00:51 -08001443 nodemask_t task_nodes;
1444 int err;
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001445 nodemask_t *old;
1446 nodemask_t *new;
1447 NODEMASK_SCRATCH(scratch);
Christoph Lameter39743882006-01-08 01:00:51 -08001448
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001449 if (!scratch)
1450 return -ENOMEM;
Christoph Lameter39743882006-01-08 01:00:51 -08001451
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001452 old = &scratch->mask1;
1453 new = &scratch->mask2;
1454
1455 err = get_nodes(old, old_nodes, maxnode);
Christoph Lameter39743882006-01-08 01:00:51 -08001456 if (err)
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001457 goto out;
1458
1459 err = get_nodes(new, new_nodes, maxnode);
1460 if (err)
1461 goto out;
Christoph Lameter39743882006-01-08 01:00:51 -08001462
1463 /* Find the mm_struct */
Zeng Zhaoming55cfaa32010-12-02 14:31:13 -08001464 rcu_read_lock();
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001465 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter39743882006-01-08 01:00:51 -08001466 if (!task) {
Zeng Zhaoming55cfaa32010-12-02 14:31:13 -08001467 rcu_read_unlock();
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001468 err = -ESRCH;
1469 goto out;
Christoph Lameter39743882006-01-08 01:00:51 -08001470 }
Christoph Lameter3268c632012-03-21 16:34:06 -07001471 get_task_struct(task);
Christoph Lameter39743882006-01-08 01:00:51 -08001472
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001473 err = -EINVAL;
Christoph Lameter39743882006-01-08 01:00:51 -08001474
1475 /*
Otto Ebeling31367462017-11-15 17:38:14 -08001476 * Check if this process has the right to modify the specified process.
1477 * Use the regular "ptrace_may_access()" checks.
Christoph Lameter39743882006-01-08 01:00:51 -08001478 */
Otto Ebeling31367462017-11-15 17:38:14 -08001479 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
David Howellsc69e8d92008-11-14 10:39:19 +11001480 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001481 err = -EPERM;
Christoph Lameter3268c632012-03-21 16:34:06 -07001482 goto out_put;
Christoph Lameter39743882006-01-08 01:00:51 -08001483 }
David Howellsc69e8d92008-11-14 10:39:19 +11001484 rcu_read_unlock();
Christoph Lameter39743882006-01-08 01:00:51 -08001485
1486 task_nodes = cpuset_mems_allowed(task);
1487 /* Is the user allowed to access the target nodes? */
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001488 if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
Christoph Lameter39743882006-01-08 01:00:51 -08001489 err = -EPERM;
Christoph Lameter3268c632012-03-21 16:34:06 -07001490 goto out_put;
Christoph Lameter39743882006-01-08 01:00:51 -08001491 }
1492
Yisheng Xie0486a382018-01-31 16:16:15 -08001493 task_nodes = cpuset_mems_allowed(current);
1494 nodes_and(*new, *new, task_nodes);
1495 if (nodes_empty(*new))
Christoph Lameter3268c632012-03-21 16:34:06 -07001496 goto out_put;
Yisheng Xie0486a382018-01-31 16:16:15 -08001497
1498 nodes_and(*new, *new, node_states[N_MEMORY]);
1499 if (nodes_empty(*new))
1500 goto out_put;
Christoph Lameter3b42d282007-08-31 00:12:08 -07001501
David Quigley86c3a762006-06-23 02:04:02 -07001502 err = security_task_movememory(task);
1503 if (err)
Christoph Lameter3268c632012-03-21 16:34:06 -07001504 goto out_put;
David Quigley86c3a762006-06-23 02:04:02 -07001505
Christoph Lameter3268c632012-03-21 16:34:06 -07001506 mm = get_task_mm(task);
1507 put_task_struct(task);
Sasha Levinf2a9ef82012-04-25 16:01:52 -07001508
1509 if (!mm) {
Christoph Lameter3268c632012-03-21 16:34:06 -07001510 err = -EINVAL;
Sasha Levinf2a9ef82012-04-25 16:01:52 -07001511 goto out;
1512 }
1513
1514 err = do_migrate_pages(mm, old, new,
1515 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
Christoph Lameter3268c632012-03-21 16:34:06 -07001516
1517 mmput(mm);
1518out:
KOSAKI Motohiro596d7cf2010-08-09 17:19:01 -07001519 NODEMASK_SCRATCH_FREE(scratch);
1520
Christoph Lameter39743882006-01-08 01:00:51 -08001521 return err;
Christoph Lameter3268c632012-03-21 16:34:06 -07001522
1523out_put:
1524 put_task_struct(task);
1525 goto out;
1526
Christoph Lameter39743882006-01-08 01:00:51 -08001527}
1528
Dominik Brodowskib6e9b0b2018-03-17 16:00:25 +01001529SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1530 const unsigned long __user *, old_nodes,
1531 const unsigned long __user *, new_nodes)
1532{
1533 return kernel_migrate_pages(pid, maxnode, old_nodes, new_nodes);
1534}
1535
Christoph Lameter39743882006-01-08 01:00:51 -08001536
Christoph Lameter8bccd852005-10-29 18:16:59 -07001537/* Retrieve NUMA policy */
Dominik Brodowskiaf03c4a2018-03-17 16:20:01 +01001538static int kernel_get_mempolicy(int __user *policy,
1539 unsigned long __user *nmask,
1540 unsigned long maxnode,
1541 unsigned long addr,
1542 unsigned long flags)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001543{
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07001544 int err;
1545 int uninitialized_var(pval);
Christoph Lameter8bccd852005-10-29 18:16:59 -07001546 nodemask_t nodes;
1547
Ralph Campbell050c17f2019-02-20 22:18:58 -08001548 if (nmask != NULL && maxnode < nr_node_ids)
Christoph Lameter8bccd852005-10-29 18:16:59 -07001549 return -EINVAL;
1550
1551 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1552
1553 if (err)
1554 return err;
1555
1556 if (policy && put_user(pval, policy))
1557 return -EFAULT;
1558
1559 if (nmask)
1560 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1561
1562 return err;
1563}
1564
Dominik Brodowskiaf03c4a2018-03-17 16:20:01 +01001565SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1566 unsigned long __user *, nmask, unsigned long, maxnode,
1567 unsigned long, addr, unsigned long, flags)
1568{
1569 return kernel_get_mempolicy(policy, nmask, maxnode, addr, flags);
1570}
1571
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572#ifdef CONFIG_COMPAT
1573
Heiko Carstensc93e0f62014-03-03 16:32:26 +01001574COMPAT_SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1575 compat_ulong_t __user *, nmask,
1576 compat_ulong_t, maxnode,
1577 compat_ulong_t, addr, compat_ulong_t, flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578{
1579 long err;
1580 unsigned long __user *nm = NULL;
1581 unsigned long nr_bits, alloc_size;
1582 DECLARE_BITMAP(bm, MAX_NUMNODES);
1583
Ralph Campbell050c17f2019-02-20 22:18:58 -08001584 nr_bits = min_t(unsigned long, maxnode-1, nr_node_ids);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1586
1587 if (nmask)
1588 nm = compat_alloc_user_space(alloc_size);
1589
Dominik Brodowskiaf03c4a2018-03-17 16:20:01 +01001590 err = kernel_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591
1592 if (!err && nmask) {
KAMEZAWA Hiroyuki2bbff6c2011-09-14 16:21:02 -07001593 unsigned long copy_size;
1594 copy_size = min_t(unsigned long, sizeof(bm), alloc_size);
1595 err = copy_from_user(bm, nm, copy_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596 /* ensure entire bitmap is zeroed */
1597 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1598 err |= compat_put_bitmap(nmask, bm, nr_bits);
1599 }
1600
1601 return err;
1602}
1603
Heiko Carstensc93e0f62014-03-03 16:32:26 +01001604COMPAT_SYSCALL_DEFINE3(set_mempolicy, int, mode, compat_ulong_t __user *, nmask,
1605 compat_ulong_t, maxnode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607 unsigned long __user *nm = NULL;
1608 unsigned long nr_bits, alloc_size;
1609 DECLARE_BITMAP(bm, MAX_NUMNODES);
1610
1611 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1612 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1613
1614 if (nmask) {
Chris Sallscf01fb92017-04-07 23:48:11 -07001615 if (compat_get_bitmap(bm, nmask, nr_bits))
1616 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 nm = compat_alloc_user_space(alloc_size);
Chris Sallscf01fb92017-04-07 23:48:11 -07001618 if (copy_to_user(nm, bm, alloc_size))
1619 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620 }
1621
Dominik Brodowskiaf03c4a2018-03-17 16:20:01 +01001622 return kernel_set_mempolicy(mode, nm, nr_bits+1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623}
1624
Heiko Carstensc93e0f62014-03-03 16:32:26 +01001625COMPAT_SYSCALL_DEFINE6(mbind, compat_ulong_t, start, compat_ulong_t, len,
1626 compat_ulong_t, mode, compat_ulong_t __user *, nmask,
1627 compat_ulong_t, maxnode, compat_ulong_t, flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 unsigned long __user *nm = NULL;
1630 unsigned long nr_bits, alloc_size;
Andi Kleendfcd3c02005-10-29 18:15:48 -07001631 nodemask_t bm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632
1633 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1634 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1635
1636 if (nmask) {
Chris Sallscf01fb92017-04-07 23:48:11 -07001637 if (compat_get_bitmap(nodes_addr(bm), nmask, nr_bits))
1638 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639 nm = compat_alloc_user_space(alloc_size);
Chris Sallscf01fb92017-04-07 23:48:11 -07001640 if (copy_to_user(nm, nodes_addr(bm), alloc_size))
1641 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642 }
1643
Dominik Brodowskie7dc9ad62018-03-17 16:12:22 +01001644 return kernel_mbind(start, len, mode, nm, nr_bits+1, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645}
1646
Dominik Brodowskib6e9b0b2018-03-17 16:00:25 +01001647COMPAT_SYSCALL_DEFINE4(migrate_pages, compat_pid_t, pid,
1648 compat_ulong_t, maxnode,
1649 const compat_ulong_t __user *, old_nodes,
1650 const compat_ulong_t __user *, new_nodes)
1651{
1652 unsigned long __user *old = NULL;
1653 unsigned long __user *new = NULL;
1654 nodemask_t tmp_mask;
1655 unsigned long nr_bits;
1656 unsigned long size;
1657
1658 nr_bits = min_t(unsigned long, maxnode - 1, MAX_NUMNODES);
1659 size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1660 if (old_nodes) {
1661 if (compat_get_bitmap(nodes_addr(tmp_mask), old_nodes, nr_bits))
1662 return -EFAULT;
1663 old = compat_alloc_user_space(new_nodes ? size * 2 : size);
1664 if (new_nodes)
1665 new = old + size / sizeof(unsigned long);
1666 if (copy_to_user(old, nodes_addr(tmp_mask), size))
1667 return -EFAULT;
1668 }
1669 if (new_nodes) {
1670 if (compat_get_bitmap(nodes_addr(tmp_mask), new_nodes, nr_bits))
1671 return -EFAULT;
1672 if (new == NULL)
1673 new = compat_alloc_user_space(size);
1674 if (copy_to_user(new, nodes_addr(tmp_mask), size))
1675 return -EFAULT;
1676 }
1677 return kernel_migrate_pages(pid, nr_bits + 1, old, new);
1678}
1679
1680#endif /* CONFIG_COMPAT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001682struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1683 unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684{
Oleg Nesterov8d902742014-10-09 15:27:45 -07001685 struct mempolicy *pol = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001686
1687 if (vma) {
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001688 if (vma->vm_ops && vma->vm_ops->get_policy) {
Oleg Nesterov8d902742014-10-09 15:27:45 -07001689 pol = vma->vm_ops->get_policy(vma, addr);
Mel Gorman00442ad2012-10-08 16:29:20 -07001690 } else if (vma->vm_policy) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001691 pol = vma->vm_policy;
Mel Gorman00442ad2012-10-08 16:29:20 -07001692
1693 /*
1694 * shmem_alloc_page() passes MPOL_F_SHARED policy with
1695 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1696 * count on these policies which will be dropped by
1697 * mpol_cond_put() later
1698 */
1699 if (mpol_needs_cond_ref(pol))
1700 mpol_get(pol);
1701 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702 }
Oleg Nesterovf15ca782014-10-09 15:27:43 -07001703
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001704 return pol;
1705}
1706
1707/*
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001708 * get_vma_policy(@vma, @addr)
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001709 * @vma: virtual memory area whose policy is sought
1710 * @addr: address in @vma for shared policy lookup
1711 *
1712 * Returns effective policy for a VMA at specified address.
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001713 * Falls back to current->mempolicy or system default policy, as necessary.
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001714 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1715 * count--added by the get_policy() vm_op, as appropriate--to protect against
1716 * freeing by another task. It is the caller's responsibility to free the
1717 * extra reference for shared policies.
1718 */
David Rientjes2f0799a2018-12-05 15:45:54 -08001719static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001720 unsigned long addr)
Oleg Nesterov74d2c3a2014-10-09 15:27:50 -07001721{
1722 struct mempolicy *pol = __get_vma_policy(vma, addr);
1723
Oleg Nesterov8d902742014-10-09 15:27:45 -07001724 if (!pol)
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001725 pol = get_task_policy(current);
Oleg Nesterov8d902742014-10-09 15:27:45 -07001726
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727 return pol;
1728}
1729
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001730bool vma_policy_mof(struct vm_area_struct *vma)
Mel Gormanfc3147242013-10-07 11:29:09 +01001731{
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001732 struct mempolicy *pol;
Oleg Nesterovf15ca782014-10-09 15:27:43 -07001733
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001734 if (vma->vm_ops && vma->vm_ops->get_policy) {
1735 bool ret = false;
Mel Gormanfc3147242013-10-07 11:29:09 +01001736
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001737 pol = vma->vm_ops->get_policy(vma, vma->vm_start);
1738 if (pol && (pol->flags & MPOL_F_MOF))
1739 ret = true;
1740 mpol_cond_put(pol);
Mel Gormanfc3147242013-10-07 11:29:09 +01001741
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001742 return ret;
Mel Gormanfc3147242013-10-07 11:29:09 +01001743 }
1744
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001745 pol = vma->vm_policy;
Oleg Nesterov8d902742014-10-09 15:27:45 -07001746 if (!pol)
Oleg Nesterov6b6482b2014-10-09 15:27:48 -07001747 pol = get_task_policy(current);
Oleg Nesterov8d902742014-10-09 15:27:45 -07001748
Mel Gormanfc3147242013-10-07 11:29:09 +01001749 return pol->flags & MPOL_F_MOF;
1750}
1751
Lai Jiangshand3eb1572013-02-22 16:33:22 -08001752static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1753{
1754 enum zone_type dynamic_policy_zone = policy_zone;
1755
1756 BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1757
1758 /*
1759 * if policy->v.nodes has movable memory only,
1760 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1761 *
1762 * policy->v.nodes is intersect with node_states[N_MEMORY].
1763 * so if the following test faile, it implies
1764 * policy->v.nodes has movable memory only.
1765 */
1766 if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY]))
1767 dynamic_policy_zone = ZONE_MOVABLE;
1768
1769 return zone >= dynamic_policy_zone;
1770}
1771
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001772/*
1773 * Return a nodemask representing a mempolicy for filtering nodes for
1774 * page allocation
1775 */
1776static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
Mel Gorman19770b32008-04-28 02:12:18 -07001777{
1778 /* Lower zones don't get a nodemask applied for MPOL_BIND */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07001779 if (unlikely(policy->mode == MPOL_BIND) &&
Lai Jiangshand3eb1572013-02-22 16:33:22 -08001780 apply_policy_zone(policy, gfp_zone(gfp)) &&
Mel Gorman19770b32008-04-28 02:12:18 -07001781 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1782 return &policy->v.nodes;
1783
1784 return NULL;
1785}
1786
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001787/* Return the node id preferred by the given mempolicy, or the given id */
1788static int policy_node(gfp_t gfp, struct mempolicy *policy,
1789 int nd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790{
Michal Hocko6d840952016-12-12 16:42:23 -08001791 if (policy->mode == MPOL_PREFERRED && !(policy->flags & MPOL_F_LOCAL))
1792 nd = policy->v.preferred_node;
1793 else {
Mel Gorman19770b32008-04-28 02:12:18 -07001794 /*
Michal Hocko6d840952016-12-12 16:42:23 -08001795 * __GFP_THISNODE shouldn't even be used with the bind policy
1796 * because we might easily break the expectation to stay on the
1797 * requested node and not break the policy.
Mel Gorman19770b32008-04-28 02:12:18 -07001798 */
Michal Hocko6d840952016-12-12 16:42:23 -08001799 WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800 }
Michal Hocko6d840952016-12-12 16:42:23 -08001801
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001802 return nd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803}
1804
1805/* Do dynamic interleaving for a process */
1806static unsigned interleave_nodes(struct mempolicy *policy)
1807{
Vlastimil Babka45816682017-07-06 15:39:59 -07001808 unsigned next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001809 struct task_struct *me = current;
1810
Vlastimil Babka45816682017-07-06 15:39:59 -07001811 next = next_node_in(me->il_prev, policy->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001812 if (next < MAX_NUMNODES)
Vlastimil Babka45816682017-07-06 15:39:59 -07001813 me->il_prev = next;
1814 return next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815}
1816
Christoph Lameterdc85da12006-01-18 17:42:36 -08001817/*
1818 * Depending on the memory policy provide a node from which to allocate the
1819 * next slab entry.
1820 */
David Rientjes2a389612014-04-07 15:37:29 -07001821unsigned int mempolicy_slab_node(void)
Christoph Lameterdc85da12006-01-18 17:42:36 -08001822{
Andi Kleene7b691b2012-06-09 02:40:03 -07001823 struct mempolicy *policy;
David Rientjes2a389612014-04-07 15:37:29 -07001824 int node = numa_mem_id();
Andi Kleene7b691b2012-06-09 02:40:03 -07001825
1826 if (in_interrupt())
David Rientjes2a389612014-04-07 15:37:29 -07001827 return node;
Andi Kleene7b691b2012-06-09 02:40:03 -07001828
1829 policy = current->mempolicy;
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001830 if (!policy || policy->flags & MPOL_F_LOCAL)
David Rientjes2a389612014-04-07 15:37:29 -07001831 return node;
Christoph Lameter765c4502006-09-27 01:50:08 -07001832
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001833 switch (policy->mode) {
1834 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07001835 /*
1836 * handled MPOL_F_LOCAL above
1837 */
1838 return policy->v.preferred_node;
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001839
Christoph Lameterdc85da12006-01-18 17:42:36 -08001840 case MPOL_INTERLEAVE:
1841 return interleave_nodes(policy);
1842
Mel Gormandd1a2392008-04-28 02:12:17 -07001843 case MPOL_BIND: {
Mel Gormanc33d6c02016-05-19 17:14:10 -07001844 struct zoneref *z;
1845
Christoph Lameterdc85da12006-01-18 17:42:36 -08001846 /*
1847 * Follow bind policy behavior and start allocation at the
1848 * first node.
1849 */
Mel Gorman19770b32008-04-28 02:12:18 -07001850 struct zonelist *zonelist;
Mel Gorman19770b32008-04-28 02:12:18 -07001851 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
Aneesh Kumar K.Vc9634cf2016-10-07 16:59:12 -07001852 zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
Mel Gormanc33d6c02016-05-19 17:14:10 -07001853 z = first_zones_zonelist(zonelist, highest_zoneidx,
1854 &policy->v.nodes);
Pavel Tatashinc1093b72018-08-21 21:53:32 -07001855 return z->zone ? zone_to_nid(z->zone) : node;
Mel Gormandd1a2392008-04-28 02:12:17 -07001856 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08001857
Christoph Lameterdc85da12006-01-18 17:42:36 -08001858 default:
Lee Schermerhornbea904d2008-04-28 02:13:18 -07001859 BUG();
Christoph Lameterdc85da12006-01-18 17:42:36 -08001860 }
1861}
1862
Andrew Mortonfee83b32016-05-19 17:11:43 -07001863/*
1864 * Do static interleaving for a VMA with known offset @n. Returns the n'th
1865 * node in pol->v.nodes (starting from n=0), wrapping around if n exceeds the
1866 * number of present nodes.
1867 */
Laurent Dufour98c70ba2017-09-08 16:12:39 -07001868static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869{
Andi Kleendfcd3c02005-10-29 18:15:48 -07001870 unsigned nnodes = nodes_weight(pol->v.nodes);
David Rientjesf5b087b2008-04-28 02:12:27 -07001871 unsigned target;
Andrew Mortonfee83b32016-05-19 17:11:43 -07001872 int i;
1873 int nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874
David Rientjesf5b087b2008-04-28 02:12:27 -07001875 if (!nnodes)
1876 return numa_node_id();
Andrew Mortonfee83b32016-05-19 17:11:43 -07001877 target = (unsigned int)n % nnodes;
1878 nid = first_node(pol->v.nodes);
1879 for (i = 0; i < target; i++)
Andi Kleendfcd3c02005-10-29 18:15:48 -07001880 nid = next_node(nid, pol->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881 return nid;
1882}
1883
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001884/* Determine a node number for interleave */
1885static inline unsigned interleave_nid(struct mempolicy *pol,
1886 struct vm_area_struct *vma, unsigned long addr, int shift)
1887{
1888 if (vma) {
1889 unsigned long off;
1890
Nishanth Aravamudan3b98b082006-08-31 21:27:53 -07001891 /*
1892 * for small pages, there is no difference between
1893 * shift and PAGE_SHIFT, so the bit-shift is safe.
1894 * for huge pages, since vm_pgoff is in units of small
1895 * pages, we need to shift off the always 0 bits to get
1896 * a useful offset.
1897 */
1898 BUG_ON(shift < PAGE_SHIFT);
1899 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001900 off += (addr - vma->vm_start) >> shift;
Laurent Dufour98c70ba2017-09-08 16:12:39 -07001901 return offset_il_node(pol, off);
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001902 } else
1903 return interleave_nodes(pol);
1904}
1905
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001906#ifdef CONFIG_HUGETLBFS
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001907/*
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001908 * huge_node(@vma, @addr, @gfp_flags, @mpol)
Fabian Frederickb46e14a2014-06-04 16:08:18 -07001909 * @vma: virtual memory area whose policy is sought
1910 * @addr: address in @vma for shared policy lookup and interleave policy
1911 * @gfp_flags: for requested zone
1912 * @mpol: pointer to mempolicy pointer for reference counted mempolicy
1913 * @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001914 *
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001915 * Returns a nid suitable for a huge page allocation and a pointer
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001916 * to the struct mempolicy for conditional unref after allocation.
1917 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1918 * @nodemask for filtering the zonelist.
Miao Xiec0ff7452010-05-24 14:32:08 -07001919 *
Mel Gormand26914d2014-04-03 14:47:24 -07001920 * Must be protected by read_mems_allowed_begin()
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001921 */
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001922int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags,
1923 struct mempolicy **mpol, nodemask_t **nodemask)
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001924{
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001925 int nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001926
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07001927 *mpol = get_vma_policy(vma, addr);
Mel Gorman19770b32008-04-28 02:12:18 -07001928 *nodemask = NULL; /* assume !MPOL_BIND */
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001929
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001930 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001931 nid = interleave_nid(*mpol, vma, addr,
1932 huge_page_shift(hstate_vma(vma)));
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001933 } else {
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001934 nid = policy_node(gfp_flags, *mpol, numa_node_id());
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07001935 if ((*mpol)->mode == MPOL_BIND)
1936 *nodemask = &(*mpol)->v.nodes;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -07001937 }
Vlastimil Babka04ec6262017-07-06 15:40:03 -07001938 return nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001939}
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001940
1941/*
1942 * init_nodemask_of_mempolicy
1943 *
1944 * If the current task's mempolicy is "default" [NULL], return 'false'
1945 * to indicate default policy. Otherwise, extract the policy nodemask
1946 * for 'bind' or 'interleave' policy into the argument nodemask, or
1947 * initialize the argument nodemask to contain the single node for
1948 * 'preferred' or 'local' policy and return 'true' to indicate presence
1949 * of non-default mempolicy.
1950 *
1951 * We don't bother with reference counting the mempolicy [mpol_get/put]
1952 * because the current task is examining it's own mempolicy and a task's
1953 * mempolicy is only ever changed by the task itself.
1954 *
1955 * N.B., it is the caller's responsibility to free a returned nodemask.
1956 */
1957bool init_nodemask_of_mempolicy(nodemask_t *mask)
1958{
1959 struct mempolicy *mempolicy;
1960 int nid;
1961
1962 if (!(mask && current->mempolicy))
1963 return false;
1964
Miao Xiec0ff7452010-05-24 14:32:08 -07001965 task_lock(current);
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001966 mempolicy = current->mempolicy;
1967 switch (mempolicy->mode) {
1968 case MPOL_PREFERRED:
1969 if (mempolicy->flags & MPOL_F_LOCAL)
1970 nid = numa_node_id();
1971 else
1972 nid = mempolicy->v.preferred_node;
1973 init_nodemask_of_node(mask, nid);
1974 break;
1975
1976 case MPOL_BIND:
1977 /* Fall through */
1978 case MPOL_INTERLEAVE:
1979 *mask = mempolicy->v.nodes;
1980 break;
1981
1982 default:
1983 BUG();
1984 }
Miao Xiec0ff7452010-05-24 14:32:08 -07001985 task_unlock(current);
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001986
1987 return true;
1988}
Chen, Kenneth W00ac59a2006-02-03 21:51:14 +01001989#endif
Christoph Lameter5da7ca82006-01-06 00:10:46 -08001990
David Rientjes6f48d0eb2010-08-09 17:18:52 -07001991/*
1992 * mempolicy_nodemask_intersects
1993 *
1994 * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
1995 * policy. Otherwise, check for intersection between mask and the policy
1996 * nodemask for 'bind' or 'interleave' policy. For 'perferred' or 'local'
1997 * policy, always return true since it may allocate elsewhere on fallback.
1998 *
1999 * Takes task_lock(tsk) to prevent freeing of its mempolicy.
2000 */
2001bool mempolicy_nodemask_intersects(struct task_struct *tsk,
2002 const nodemask_t *mask)
2003{
2004 struct mempolicy *mempolicy;
2005 bool ret = true;
2006
2007 if (!mask)
2008 return ret;
2009 task_lock(tsk);
2010 mempolicy = tsk->mempolicy;
2011 if (!mempolicy)
2012 goto out;
2013
2014 switch (mempolicy->mode) {
2015 case MPOL_PREFERRED:
2016 /*
2017 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
2018 * allocate from, they may fallback to other nodes when oom.
2019 * Thus, it's possible for tsk to have allocated memory from
2020 * nodes in mask.
2021 */
2022 break;
2023 case MPOL_BIND:
2024 case MPOL_INTERLEAVE:
2025 ret = nodes_intersects(mempolicy->v.nodes, *mask);
2026 break;
2027 default:
2028 BUG();
2029 }
2030out:
2031 task_unlock(tsk);
2032 return ret;
2033}
2034
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035/* Allocate a page in interleaved policy.
2036 Own path because it needs to do special accounting. */
Andi Kleen662f3a02005-10-29 18:15:49 -07002037static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
2038 unsigned nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002040 struct page *page;
2041
Vlastimil Babka04ec6262017-07-06 15:40:03 -07002042 page = __alloc_pages(gfp, order, nid);
Kemi Wang45180852017-11-15 17:38:22 -08002043 /* skip NUMA_INTERLEAVE_HIT counter update if numa stats is disabled */
2044 if (!static_branch_likely(&vm_numa_stat_key))
2045 return page;
Andrey Ryabininde55c8b2017-10-13 15:57:43 -07002046 if (page && page_to_nid(page) == nid) {
2047 preempt_disable();
2048 __inc_numa_state(page_zone(page), NUMA_INTERLEAVE_HIT);
2049 preempt_enable();
2050 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002051 return page;
2052}
2053
2054/**
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002055 * alloc_pages_vma - Allocate a page for a VMA.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056 *
2057 * @gfp:
2058 * %GFP_USER user allocation.
2059 * %GFP_KERNEL kernel allocations,
2060 * %GFP_HIGHMEM highmem/user allocations,
2061 * %GFP_FS allocation should not call back into a file system.
2062 * %GFP_ATOMIC don't sleep.
2063 *
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002064 * @order:Order of the GFP allocation.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065 * @vma: Pointer to VMA or NULL if not available.
2066 * @addr: Virtual Address of the allocation. Must be inside the VMA.
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002067 * @node: Which node to prefer for allocation (modulo policy).
David Rientjes356ff8a2018-12-07 14:50:16 -08002068 * @hugepage: for hugepages try only the preferred node if possible
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069 *
2070 * This function allocates a page from the kernel page pool and applies
2071 * a NUMA policy associated with the VMA or the current process.
2072 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
2073 * mm_struct of the VMA to prevent it from going away. Should be used for
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002074 * all allocations for pages that will be mapped into user space. Returns
2075 * NULL when no page can be allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076 */
2077struct page *
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002078alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
David Rientjes356ff8a2018-12-07 14:50:16 -08002079 unsigned long addr, int node, bool hugepage)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080{
Mel Gormancc9a6c82012-03-21 16:34:11 -07002081 struct mempolicy *pol;
Miao Xiec0ff7452010-05-24 14:32:08 -07002082 struct page *page;
Vlastimil Babka04ec6262017-07-06 15:40:03 -07002083 int preferred_nid;
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002084 nodemask_t *nmask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002085
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07002086 pol = get_vma_policy(vma, addr);
Mel Gormancc9a6c82012-03-21 16:34:11 -07002087
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002088 if (pol->mode == MPOL_INTERLEAVE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002089 unsigned nid;
Christoph Lameter5da7ca82006-01-06 00:10:46 -08002090
Andi Kleen8eac5632011-02-25 14:44:28 -08002091 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002092 mpol_cond_put(pol);
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002093 page = alloc_page_interleave(gfp, order, nid);
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002094 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095 }
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002096
David Rientjes356ff8a2018-12-07 14:50:16 -08002097 if (unlikely(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && hugepage)) {
2098 int hpage_node = node;
2099
2100 /*
2101 * For hugepage allocation and non-interleave policy which
2102 * allows the current node (or other explicitly preferred
2103 * node) we only try to allocate from the current/preferred
2104 * node and don't fall back to other nodes, as the cost of
2105 * remote accesses would likely offset THP benefits.
2106 *
2107 * If the policy is interleave, or does not allow the current
2108 * node in its nodemask, we allocate the standard way.
2109 */
2110 if (pol->mode == MPOL_PREFERRED && !(pol->flags & MPOL_F_LOCAL))
2111 hpage_node = pol->v.preferred_node;
2112
2113 nmask = policy_nodemask(gfp, pol);
2114 if (!nmask || node_isset(hpage_node, *nmask)) {
2115 mpol_cond_put(pol);
2116 page = __alloc_pages_node(hpage_node,
2117 gfp | __GFP_THISNODE, order);
2118 goto out;
2119 }
2120 }
2121
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002122 nmask = policy_nodemask(gfp, pol);
Vlastimil Babka04ec6262017-07-06 15:40:03 -07002123 preferred_nid = policy_node(gfp, pol, node);
2124 page = __alloc_pages_nodemask(gfp, order, preferred_nid, nmask);
Vlastimil Babkad51e9892017-01-24 15:18:18 -08002125 mpol_cond_put(pol);
Vlastimil Babkabe97a412015-02-11 15:27:15 -08002126out:
Miao Xiec0ff7452010-05-24 14:32:08 -07002127 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002128}
Christoph Hellwig69262212019-06-26 14:27:05 +02002129EXPORT_SYMBOL(alloc_pages_vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002130
2131/**
2132 * alloc_pages_current - Allocate pages.
2133 *
2134 * @gfp:
2135 * %GFP_USER user allocation,
2136 * %GFP_KERNEL kernel allocation,
2137 * %GFP_HIGHMEM highmem allocation,
2138 * %GFP_FS don't call back into a file system.
2139 * %GFP_ATOMIC don't sleep.
2140 * @order: Power of two of allocation size in pages. 0 is a single page.
2141 *
2142 * Allocate a page from the kernel page pool. When not in
2143 * interrupt context and apply the current process NUMA policy.
2144 * Returns NULL when no page can be allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145 */
Al Virodd0fc662005-10-07 07:46:04 +01002146struct page *alloc_pages_current(gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002147{
Oleg Nesterov8d902742014-10-09 15:27:45 -07002148 struct mempolicy *pol = &default_policy;
Miao Xiec0ff7452010-05-24 14:32:08 -07002149 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002150
Oleg Nesterov8d902742014-10-09 15:27:45 -07002151 if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2152 pol = get_task_policy(current);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -07002153
2154 /*
2155 * No reference counting needed for current->mempolicy
2156 * nor system default_policy
2157 */
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002158 if (pol->mode == MPOL_INTERLEAVE)
Miao Xiec0ff7452010-05-24 14:32:08 -07002159 page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2160 else
2161 page = __alloc_pages_nodemask(gfp, order,
Vlastimil Babka04ec6262017-07-06 15:40:03 -07002162 policy_node(gfp, pol, numa_node_id()),
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002163 policy_nodemask(gfp, pol));
Mel Gormancc9a6c82012-03-21 16:34:11 -07002164
Miao Xiec0ff7452010-05-24 14:32:08 -07002165 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002166}
2167EXPORT_SYMBOL(alloc_pages_current);
2168
Oleg Nesterovef0855d2013-09-11 14:20:14 -07002169int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2170{
2171 struct mempolicy *pol = mpol_dup(vma_policy(src));
2172
2173 if (IS_ERR(pol))
2174 return PTR_ERR(pol);
2175 dst->vm_policy = pol;
2176 return 0;
2177}
2178
Paul Jackson42253992006-01-08 01:01:59 -08002179/*
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07002180 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
Paul Jackson42253992006-01-08 01:01:59 -08002181 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2182 * with the mems_allowed returned by cpuset_mems_allowed(). This
2183 * keeps mempolicies cpuset relative after its cpuset moves. See
2184 * further kernel/cpuset.c update_nodemask().
Miao Xie708c1bb2010-05-24 14:32:07 -07002185 *
2186 * current's mempolicy may be rebinded by the other task(the task that changes
2187 * cpuset's mems), so we needn't do rebind work for current task.
Paul Jackson42253992006-01-08 01:01:59 -08002188 */
Paul Jackson42253992006-01-08 01:01:59 -08002189
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07002190/* Slow path of a mempolicy duplicate */
2191struct mempolicy *__mpol_dup(struct mempolicy *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002192{
2193 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2194
2195 if (!new)
2196 return ERR_PTR(-ENOMEM);
Miao Xie708c1bb2010-05-24 14:32:07 -07002197
2198 /* task's mempolicy is protected by alloc_lock */
2199 if (old == current->mempolicy) {
2200 task_lock(current);
2201 *new = *old;
2202 task_unlock(current);
2203 } else
2204 *new = *old;
2205
Paul Jackson42253992006-01-08 01:01:59 -08002206 if (current_cpuset_is_being_rebound()) {
2207 nodemask_t mems = cpuset_mems_allowed(current);
Vlastimil Babka213980c2017-07-06 15:40:06 -07002208 mpol_rebind_policy(new, &mems);
Paul Jackson42253992006-01-08 01:01:59 -08002209 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002210 atomic_set(&new->refcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002211 return new;
2212}
2213
2214/* Slow path of a mempolicy comparison */
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002215bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002216{
2217 if (!a || !b)
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002218 return false;
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002219 if (a->mode != b->mode)
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002220 return false;
Bob Liu19800502010-05-24 14:32:01 -07002221 if (a->flags != b->flags)
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002222 return false;
Bob Liu19800502010-05-24 14:32:01 -07002223 if (mpol_store_user_nodemask(a))
2224 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002225 return false;
Bob Liu19800502010-05-24 14:32:01 -07002226
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002227 switch (a->mode) {
Mel Gorman19770b32008-04-28 02:12:18 -07002228 case MPOL_BIND:
2229 /* Fall through */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002230 case MPOL_INTERLEAVE:
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002231 return !!nodes_equal(a->v.nodes, b->v.nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002232 case MPOL_PREFERRED:
Yisheng Xie8970a632018-03-22 16:17:02 -07002233 /* a's ->flags is the same as b's */
2234 if (a->flags & MPOL_F_LOCAL)
2235 return true;
Namhyung Kim75719662011-03-22 16:33:02 -07002236 return a->v.preferred_node == b->v.preferred_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002237 default:
2238 BUG();
KOSAKI Motohirofcfb4dc2012-01-10 15:08:21 -08002239 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002240 }
2241}
2242
Linus Torvalds1da177e2005-04-16 15:20:36 -07002243/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244 * Shared memory backing store policy support.
2245 *
2246 * Remember policies even when nobody has shared memory mapped.
2247 * The policies are kept in Red-Black tree linked from the inode.
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002248 * They are protected by the sp->lock rwlock, which should be held
Linus Torvalds1da177e2005-04-16 15:20:36 -07002249 * for any accesses to the tree.
2250 */
2251
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002252/*
2253 * lookup first element intersecting start-end. Caller holds sp->lock for
2254 * reading or for writing
2255 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002256static struct sp_node *
2257sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2258{
2259 struct rb_node *n = sp->root.rb_node;
2260
2261 while (n) {
2262 struct sp_node *p = rb_entry(n, struct sp_node, nd);
2263
2264 if (start >= p->end)
2265 n = n->rb_right;
2266 else if (end <= p->start)
2267 n = n->rb_left;
2268 else
2269 break;
2270 }
2271 if (!n)
2272 return NULL;
2273 for (;;) {
2274 struct sp_node *w = NULL;
2275 struct rb_node *prev = rb_prev(n);
2276 if (!prev)
2277 break;
2278 w = rb_entry(prev, struct sp_node, nd);
2279 if (w->end <= start)
2280 break;
2281 n = prev;
2282 }
2283 return rb_entry(n, struct sp_node, nd);
2284}
2285
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002286/*
2287 * Insert a new shared policy into the list. Caller holds sp->lock for
2288 * writing.
2289 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002290static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2291{
2292 struct rb_node **p = &sp->root.rb_node;
2293 struct rb_node *parent = NULL;
2294 struct sp_node *nd;
2295
2296 while (*p) {
2297 parent = *p;
2298 nd = rb_entry(parent, struct sp_node, nd);
2299 if (new->start < nd->start)
2300 p = &(*p)->rb_left;
2301 else if (new->end > nd->end)
2302 p = &(*p)->rb_right;
2303 else
2304 BUG();
2305 }
2306 rb_link_node(&new->nd, parent, p);
2307 rb_insert_color(&new->nd, &sp->root);
Paul Mundt140d5a42007-07-15 23:38:16 -07002308 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002309 new->policy ? new->policy->mode : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002310}
2311
2312/* Find shared policy intersecting idx */
2313struct mempolicy *
2314mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2315{
2316 struct mempolicy *pol = NULL;
2317 struct sp_node *sn;
2318
2319 if (!sp->root.rb_node)
2320 return NULL;
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002321 read_lock(&sp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002322 sn = sp_lookup(sp, idx, idx+1);
2323 if (sn) {
2324 mpol_get(sn->policy);
2325 pol = sn->policy;
2326 }
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002327 read_unlock(&sp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002328 return pol;
2329}
2330
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002331static void sp_free(struct sp_node *n)
2332{
2333 mpol_put(n->policy);
2334 kmem_cache_free(sn_cache, n);
2335}
2336
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002337/**
2338 * mpol_misplaced - check whether current page node is valid in policy
2339 *
Fabian Frederickb46e14a2014-06-04 16:08:18 -07002340 * @page: page to be checked
2341 * @vma: vm area where page mapped
2342 * @addr: virtual address where page mapped
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002343 *
2344 * Lookup current policy node id for vma,addr and "compare to" page's
2345 * node id.
2346 *
2347 * Returns:
2348 * -1 - not misplaced, page is in the right node
2349 * node - node id where the page should be
2350 *
2351 * Policy determination "mimics" alloc_page_vma().
2352 * Called from fault path where we know the vma and faulting address.
2353 */
2354int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2355{
2356 struct mempolicy *pol;
Mel Gormanc33d6c02016-05-19 17:14:10 -07002357 struct zoneref *z;
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002358 int curnid = page_to_nid(page);
2359 unsigned long pgoff;
Peter Zijlstra90572892013-10-07 11:29:20 +01002360 int thiscpu = raw_smp_processor_id();
2361 int thisnid = cpu_to_node(thiscpu);
Anshuman Khandual98fa15f2019-03-05 15:42:58 -08002362 int polnid = NUMA_NO_NODE;
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002363 int ret = -1;
2364
Oleg Nesterovdd6eecb2014-10-09 15:27:57 -07002365 pol = get_vma_policy(vma, addr);
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002366 if (!(pol->flags & MPOL_F_MOF))
2367 goto out;
2368
2369 switch (pol->mode) {
2370 case MPOL_INTERLEAVE:
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002371 pgoff = vma->vm_pgoff;
2372 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
Laurent Dufour98c70ba2017-09-08 16:12:39 -07002373 polnid = offset_il_node(pol, pgoff);
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002374 break;
2375
2376 case MPOL_PREFERRED:
2377 if (pol->flags & MPOL_F_LOCAL)
2378 polnid = numa_node_id();
2379 else
2380 polnid = pol->v.preferred_node;
2381 break;
2382
2383 case MPOL_BIND:
Mel Gormanc33d6c02016-05-19 17:14:10 -07002384
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002385 /*
2386 * allows binding to multiple nodes.
2387 * use current page if in policy nodemask,
2388 * else select nearest allowed node, if any.
2389 * If no allowed nodes, use current [!misplaced].
2390 */
2391 if (node_isset(curnid, pol->v.nodes))
2392 goto out;
Mel Gormanc33d6c02016-05-19 17:14:10 -07002393 z = first_zones_zonelist(
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002394 node_zonelist(numa_node_id(), GFP_HIGHUSER),
2395 gfp_zone(GFP_HIGHUSER),
Mel Gormanc33d6c02016-05-19 17:14:10 -07002396 &pol->v.nodes);
Pavel Tatashinc1093b72018-08-21 21:53:32 -07002397 polnid = zone_to_nid(z->zone);
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002398 break;
2399
2400 default:
2401 BUG();
2402 }
Mel Gorman5606e382012-11-02 18:19:13 +00002403
2404 /* Migrate the page towards the node whose CPU is referencing it */
Mel Gormane42c8ff2012-11-12 09:17:07 +00002405 if (pol->flags & MPOL_F_MORON) {
Peter Zijlstra90572892013-10-07 11:29:20 +01002406 polnid = thisnid;
Mel Gorman5606e382012-11-02 18:19:13 +00002407
Rik van Riel10f39042014-01-27 17:03:44 -05002408 if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
Rik van Rielde1c9ce2013-10-07 11:29:39 +01002409 goto out;
Mel Gormane42c8ff2012-11-12 09:17:07 +00002410 }
2411
Lee Schermerhorn771fb4d2012-10-25 14:16:30 +02002412 if (curnid != polnid)
2413 ret = polnid;
2414out:
2415 mpol_cond_put(pol);
2416
2417 return ret;
2418}
2419
David Rientjesc11600e2016-09-01 16:15:07 -07002420/*
2421 * Drop the (possibly final) reference to task->mempolicy. It needs to be
2422 * dropped after task->mempolicy is set to NULL so that any allocation done as
2423 * part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed
2424 * policy.
2425 */
2426void mpol_put_task_policy(struct task_struct *task)
2427{
2428 struct mempolicy *pol;
2429
2430 task_lock(task);
2431 pol = task->mempolicy;
2432 task->mempolicy = NULL;
2433 task_unlock(task);
2434 mpol_put(pol);
2435}
2436
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2438{
Paul Mundt140d5a42007-07-15 23:38:16 -07002439 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440 rb_erase(&n->nd, &sp->root);
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002441 sp_free(n);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002442}
2443
Mel Gorman42288fe2012-12-21 23:10:25 +00002444static void sp_node_init(struct sp_node *node, unsigned long start,
2445 unsigned long end, struct mempolicy *pol)
2446{
2447 node->start = start;
2448 node->end = end;
2449 node->policy = pol;
2450}
2451
Adrian Bunkdbcb0f12007-10-16 01:26:26 -07002452static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2453 struct mempolicy *pol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002454{
KOSAKI Motohiro869833f2012-10-08 16:29:16 -07002455 struct sp_node *n;
2456 struct mempolicy *newpol;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002457
KOSAKI Motohiro869833f2012-10-08 16:29:16 -07002458 n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002459 if (!n)
2460 return NULL;
KOSAKI Motohiro869833f2012-10-08 16:29:16 -07002461
2462 newpol = mpol_dup(pol);
2463 if (IS_ERR(newpol)) {
2464 kmem_cache_free(sn_cache, n);
2465 return NULL;
2466 }
2467 newpol->flags |= MPOL_F_SHARED;
Mel Gorman42288fe2012-12-21 23:10:25 +00002468 sp_node_init(n, start, end, newpol);
KOSAKI Motohiro869833f2012-10-08 16:29:16 -07002469
Linus Torvalds1da177e2005-04-16 15:20:36 -07002470 return n;
2471}
2472
2473/* Replace a policy range. */
2474static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2475 unsigned long end, struct sp_node *new)
2476{
Mel Gormanb22d1272012-10-08 16:29:17 -07002477 struct sp_node *n;
Mel Gorman42288fe2012-12-21 23:10:25 +00002478 struct sp_node *n_new = NULL;
2479 struct mempolicy *mpol_new = NULL;
Mel Gormanb22d1272012-10-08 16:29:17 -07002480 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002481
Mel Gorman42288fe2012-12-21 23:10:25 +00002482restart:
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002483 write_lock(&sp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002484 n = sp_lookup(sp, start, end);
2485 /* Take care of old policies in the same range. */
2486 while (n && n->start < end) {
2487 struct rb_node *next = rb_next(&n->nd);
2488 if (n->start >= start) {
2489 if (n->end <= end)
2490 sp_delete(sp, n);
2491 else
2492 n->start = end;
2493 } else {
2494 /* Old policy spanning whole new range. */
2495 if (n->end > end) {
Mel Gorman42288fe2012-12-21 23:10:25 +00002496 if (!n_new)
2497 goto alloc_new;
2498
2499 *mpol_new = *n->policy;
2500 atomic_set(&mpol_new->refcnt, 1);
KOSAKI Motohiro78806392013-03-08 12:43:29 -08002501 sp_node_init(n_new, end, n->end, mpol_new);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002502 n->end = start;
Hillf Danton5ca39572013-03-08 12:43:28 -08002503 sp_insert(sp, n_new);
Mel Gorman42288fe2012-12-21 23:10:25 +00002504 n_new = NULL;
2505 mpol_new = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002506 break;
2507 } else
2508 n->end = start;
2509 }
2510 if (!next)
2511 break;
2512 n = rb_entry(next, struct sp_node, nd);
2513 }
2514 if (new)
2515 sp_insert(sp, new);
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002516 write_unlock(&sp->lock);
Mel Gorman42288fe2012-12-21 23:10:25 +00002517 ret = 0;
2518
2519err_out:
2520 if (mpol_new)
2521 mpol_put(mpol_new);
2522 if (n_new)
2523 kmem_cache_free(sn_cache, n_new);
2524
Mel Gormanb22d1272012-10-08 16:29:17 -07002525 return ret;
Mel Gorman42288fe2012-12-21 23:10:25 +00002526
2527alloc_new:
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002528 write_unlock(&sp->lock);
Mel Gorman42288fe2012-12-21 23:10:25 +00002529 ret = -ENOMEM;
2530 n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2531 if (!n_new)
2532 goto err_out;
2533 mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2534 if (!mpol_new)
2535 goto err_out;
2536 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002537}
2538
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002539/**
2540 * mpol_shared_policy_init - initialize shared policy for inode
2541 * @sp: pointer to inode shared policy
2542 * @mpol: struct mempolicy to install
2543 *
2544 * Install non-NULL @mpol in inode's shared policy rb-tree.
2545 * On entry, the current task has a reference on a non-NULL @mpol.
2546 * This must be released on exit.
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002547 * This is called at get_inode() calls and we can use GFP_KERNEL.
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002548 */
2549void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
Robin Holt7339ff82006-01-14 13:20:48 -08002550{
Miao Xie58568d22009-06-16 15:31:49 -07002551 int ret;
2552
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002553 sp->root = RB_ROOT; /* empty tree == default mempolicy */
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002554 rwlock_init(&sp->lock);
Robin Holt7339ff82006-01-14 13:20:48 -08002555
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002556 if (mpol) {
2557 struct vm_area_struct pvma;
2558 struct mempolicy *new;
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002559 NODEMASK_SCRATCH(scratch);
Robin Holt7339ff82006-01-14 13:20:48 -08002560
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002561 if (!scratch)
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002562 goto put_mpol;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002563 /* contextualize the tmpfs mount point mempolicy */
2564 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
Lee Schermerhorn15d77832010-05-24 14:32:04 -07002565 if (IS_ERR(new))
Dan Carpenter0cae3452010-05-25 23:42:58 -07002566 goto free_scratch; /* no valid nodemask intersection */
Miao Xie58568d22009-06-16 15:31:49 -07002567
2568 task_lock(current);
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002569 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
Miao Xie58568d22009-06-16 15:31:49 -07002570 task_unlock(current);
Lee Schermerhorn15d77832010-05-24 14:32:04 -07002571 if (ret)
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002572 goto put_new;
Robin Holt7339ff82006-01-14 13:20:48 -08002573
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002574 /* Create pseudo-vma that contains just the policy */
Kirill A. Shutemov2c4541e2018-07-26 16:37:30 -07002575 vma_init(&pvma, NULL);
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002576 pvma.vm_end = TASK_SIZE; /* policy covers entire file */
2577 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
Lee Schermerhorn15d77832010-05-24 14:32:04 -07002578
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002579put_new:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002580 mpol_put(new); /* drop initial ref */
Dan Carpenter0cae3452010-05-25 23:42:58 -07002581free_scratch:
KAMEZAWA Hiroyuki4bfc4492009-08-06 15:07:33 -07002582 NODEMASK_SCRATCH_FREE(scratch);
Lee Schermerhorn5c0c1652010-06-29 15:05:30 -07002583put_mpol:
2584 mpol_put(mpol); /* drop our incoming ref on sb mpol */
Robin Holt7339ff82006-01-14 13:20:48 -08002585 }
2586}
2587
Linus Torvalds1da177e2005-04-16 15:20:36 -07002588int mpol_set_shared_policy(struct shared_policy *info,
2589 struct vm_area_struct *vma, struct mempolicy *npol)
2590{
2591 int err;
2592 struct sp_node *new = NULL;
2593 unsigned long sz = vma_pages(vma);
2594
David Rientjes028fec42008-04-28 02:12:25 -07002595 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002596 vma->vm_pgoff,
Lee Schermerhorn45c47452008-04-28 02:13:12 -07002597 sz, npol ? npol->mode : -1,
David Rientjes028fec42008-04-28 02:12:25 -07002598 npol ? npol->flags : -1,
David Rientjes00ef2d22013-02-22 16:35:36 -08002599 npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002600
2601 if (npol) {
2602 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2603 if (!new)
2604 return -ENOMEM;
2605 }
2606 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2607 if (err && new)
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002608 sp_free(new);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002609 return err;
2610}
2611
2612/* Free a backing policy store on inode delete. */
2613void mpol_free_shared_policy(struct shared_policy *p)
2614{
2615 struct sp_node *n;
2616 struct rb_node *next;
2617
2618 if (!p->root.rb_node)
2619 return;
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002620 write_lock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002621 next = rb_first(&p->root);
2622 while (next) {
2623 n = rb_entry(next, struct sp_node, nd);
2624 next = rb_next(&n->nd);
KOSAKI Motohiro63f74ca2012-10-08 16:29:19 -07002625 sp_delete(p, n);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002626 }
Nathan Zimmer4a8c7bb2016-01-14 15:18:36 -08002627 write_unlock(&p->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002628}
2629
Mel Gorman1a687c22012-11-22 11:16:36 +00002630#ifdef CONFIG_NUMA_BALANCING
Mel Gormanc2976632014-01-29 14:05:42 -08002631static int __initdata numabalancing_override;
Mel Gorman1a687c22012-11-22 11:16:36 +00002632
2633static void __init check_numabalancing_enable(void)
2634{
2635 bool numabalancing_default = false;
2636
2637 if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
2638 numabalancing_default = true;
2639
Mel Gormanc2976632014-01-29 14:05:42 -08002640 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2641 if (numabalancing_override)
2642 set_numabalancing_state(numabalancing_override == 1);
2643
Mel Gormanb0dc2b92015-05-14 15:17:09 -07002644 if (num_online_nodes() > 1 && !numabalancing_override) {
Joe Perches756a0252016-03-17 14:19:47 -07002645 pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
Mel Gormanc2976632014-01-29 14:05:42 -08002646 numabalancing_default ? "Enabling" : "Disabling");
Mel Gorman1a687c22012-11-22 11:16:36 +00002647 set_numabalancing_state(numabalancing_default);
2648 }
2649}
2650
2651static int __init setup_numabalancing(char *str)
2652{
2653 int ret = 0;
2654 if (!str)
2655 goto out;
Mel Gorman1a687c22012-11-22 11:16:36 +00002656
2657 if (!strcmp(str, "enable")) {
Mel Gormanc2976632014-01-29 14:05:42 -08002658 numabalancing_override = 1;
Mel Gorman1a687c22012-11-22 11:16:36 +00002659 ret = 1;
2660 } else if (!strcmp(str, "disable")) {
Mel Gormanc2976632014-01-29 14:05:42 -08002661 numabalancing_override = -1;
Mel Gorman1a687c22012-11-22 11:16:36 +00002662 ret = 1;
2663 }
2664out:
2665 if (!ret)
Andrew Morton4a404be2014-01-29 14:05:43 -08002666 pr_warn("Unable to parse numa_balancing=\n");
Mel Gorman1a687c22012-11-22 11:16:36 +00002667
2668 return ret;
2669}
2670__setup("numa_balancing=", setup_numabalancing);
2671#else
2672static inline void __init check_numabalancing_enable(void)
2673{
2674}
2675#endif /* CONFIG_NUMA_BALANCING */
2676
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677/* assumes fs == KERNEL_DS */
2678void __init numa_policy_init(void)
2679{
Paul Mundtb71636e2007-07-15 23:38:15 -07002680 nodemask_t interleave_nodes;
2681 unsigned long largest = 0;
2682 int nid, prefer = 0;
2683
Linus Torvalds1da177e2005-04-16 15:20:36 -07002684 policy_cache = kmem_cache_create("numa_policy",
2685 sizeof(struct mempolicy),
Paul Mundt20c2df82007-07-20 10:11:58 +09002686 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002687
2688 sn_cache = kmem_cache_create("shared_policy_node",
2689 sizeof(struct sp_node),
Paul Mundt20c2df82007-07-20 10:11:58 +09002690 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002691
Mel Gorman5606e382012-11-02 18:19:13 +00002692 for_each_node(nid) {
2693 preferred_node_policy[nid] = (struct mempolicy) {
2694 .refcnt = ATOMIC_INIT(1),
2695 .mode = MPOL_PREFERRED,
2696 .flags = MPOL_F_MOF | MPOL_F_MORON,
2697 .v = { .preferred_node = nid, },
2698 };
2699 }
2700
Paul Mundtb71636e2007-07-15 23:38:15 -07002701 /*
2702 * Set interleaving policy for system init. Interleaving is only
2703 * enabled across suitably sized nodes (default is >= 16MB), or
2704 * fall back to the largest node if they're all smaller.
2705 */
2706 nodes_clear(interleave_nodes);
Lai Jiangshan01f13bd2012-12-12 13:51:33 -08002707 for_each_node_state(nid, N_MEMORY) {
Paul Mundtb71636e2007-07-15 23:38:15 -07002708 unsigned long total_pages = node_present_pages(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002709
Paul Mundtb71636e2007-07-15 23:38:15 -07002710 /* Preserve the largest node */
2711 if (largest < total_pages) {
2712 largest = total_pages;
2713 prefer = nid;
2714 }
2715
2716 /* Interleave this node? */
2717 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2718 node_set(nid, interleave_nodes);
2719 }
2720
2721 /* All too small, use the largest */
2722 if (unlikely(nodes_empty(interleave_nodes)))
2723 node_set(prefer, interleave_nodes);
2724
David Rientjes028fec42008-04-28 02:12:25 -07002725 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -07002726 pr_err("%s: interleaving failed\n", __func__);
Mel Gorman1a687c22012-11-22 11:16:36 +00002727
2728 check_numabalancing_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002729}
2730
Christoph Lameter8bccd852005-10-29 18:16:59 -07002731/* Reset policy of current process to default */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002732void numa_default_policy(void)
2733{
David Rientjes028fec42008-04-28 02:12:25 -07002734 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002735}
Paul Jackson68860ec2005-10-30 15:02:36 -08002736
Paul Jackson42253992006-01-08 01:01:59 -08002737/*
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002738 * Parse and format mempolicy from/to strings
2739 */
2740
2741/*
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002742 * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag.
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002743 */
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002744static const char * const policy_modes[] =
2745{
2746 [MPOL_DEFAULT] = "default",
2747 [MPOL_PREFERRED] = "prefer",
2748 [MPOL_BIND] = "bind",
2749 [MPOL_INTERLEAVE] = "interleave",
Lee Schermerhornd3a71032012-10-25 14:16:29 +02002750 [MPOL_LOCAL] = "local",
Lee Schermerhorn345ace92010-05-24 14:32:04 -07002751};
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002752
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002753
2754#ifdef CONFIG_TMPFS
2755/**
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002756 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002757 * @str: string containing mempolicy to parse
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002758 * @mpol: pointer to struct mempolicy pointer, returned on success.
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002759 *
2760 * Format of input:
2761 * <mode>[=<flags>][:<nodelist>]
2762 *
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002763 * On success, returns 0, else 1
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002764 */
Hugh Dickinsa7a88b22013-01-02 02:04:23 -08002765int mpol_parse_str(char *str, struct mempolicy **mpol)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002766{
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002767 struct mempolicy *new = NULL;
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002768 unsigned short mode_flags;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002769 nodemask_t nodes;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002770 char *nodelist = strchr(str, ':');
2771 char *flags = strchr(str, '=');
zhong jiangdedf2c72018-10-26 15:06:57 -07002772 int err = 1, mode;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002773
2774 if (nodelist) {
2775 /* NUL-terminate mode or flags string */
2776 *nodelist++ = '\0';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002777 if (nodelist_parse(nodelist, nodes))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002778 goto out;
Lai Jiangshan01f13bd2012-12-12 13:51:33 -08002779 if (!nodes_subset(nodes, node_states[N_MEMORY]))
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002780 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002781 } else
2782 nodes_clear(nodes);
2783
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002784 if (flags)
2785 *flags++ = '\0'; /* terminate mode string */
2786
zhong jiangdedf2c72018-10-26 15:06:57 -07002787 mode = match_string(policy_modes, MPOL_MAX, str);
2788 if (mode < 0)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002789 goto out;
2790
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002791 switch (mode) {
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002792 case MPOL_PREFERRED:
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002793 /*
2794 * Insist on a nodelist of one node only
2795 */
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002796 if (nodelist) {
2797 char *rest = nodelist;
2798 while (isdigit(*rest))
2799 rest++;
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002800 if (*rest)
2801 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002802 }
2803 break;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002804 case MPOL_INTERLEAVE:
2805 /*
2806 * Default to online nodes with memory if no nodelist
2807 */
2808 if (!nodelist)
Lai Jiangshan01f13bd2012-12-12 13:51:33 -08002809 nodes = node_states[N_MEMORY];
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002810 break;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002811 case MPOL_LOCAL:
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002812 /*
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002813 * Don't allow a nodelist; mpol_new() checks flags
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002814 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002815 if (nodelist)
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002816 goto out;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002817 mode = MPOL_PREFERRED;
Lee Schermerhorn3f226aa2008-04-28 02:13:24 -07002818 break;
Ravikiran G Thirumalai413b43d2010-03-23 13:35:28 -07002819 case MPOL_DEFAULT:
2820 /*
2821 * Insist on a empty nodelist
2822 */
2823 if (!nodelist)
2824 err = 0;
2825 goto out;
KOSAKI Motohirod69b2e62010-03-23 13:35:30 -07002826 case MPOL_BIND:
2827 /*
2828 * Insist on a nodelist
2829 */
2830 if (!nodelist)
2831 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002832 }
2833
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002834 mode_flags = 0;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002835 if (flags) {
2836 /*
2837 * Currently, we only support two mutually exclusive
2838 * mode flags.
2839 */
2840 if (!strcmp(flags, "static"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002841 mode_flags |= MPOL_F_STATIC_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002842 else if (!strcmp(flags, "relative"))
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002843 mode_flags |= MPOL_F_RELATIVE_NODES;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002844 else
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002845 goto out;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002846 }
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002847
2848 new = mpol_new(mode, mode_flags, &nodes);
2849 if (IS_ERR(new))
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002850 goto out;
2851
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002852 /*
2853 * Save nodes for mpol_to_str() to show the tmpfs mount options
2854 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
2855 */
2856 if (mode != MPOL_PREFERRED)
2857 new->v.nodes = nodes;
2858 else if (nodelist)
2859 new->v.preferred_node = first_node(nodes);
2860 else
2861 new->flags |= MPOL_F_LOCAL;
2862
2863 /*
2864 * Save nodes for contextualization: this will be used to "clone"
2865 * the mempolicy in a specific context [cpuset] at a later time.
2866 */
2867 new->w.user_nodemask = nodes;
2868
KOSAKI Motohiro926f2ae2010-03-23 13:35:32 -07002869 err = 0;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002870
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002871out:
2872 /* Restore string for error message */
2873 if (nodelist)
2874 *--nodelist = ':';
2875 if (flags)
2876 *--flags = '=';
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002877 if (!err)
2878 *mpol = new;
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -07002879 return err;
2880}
2881#endif /* CONFIG_TMPFS */
2882
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002883/**
2884 * mpol_to_str - format a mempolicy structure for printing
2885 * @buffer: to contain formatted mempolicy string
2886 * @maxlen: length of @buffer
2887 * @pol: pointer to mempolicy to be formatted
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002888 *
David Rientjes948927e2013-11-12 15:07:28 -08002889 * Convert @pol into a string. If @buffer is too short, truncate the string.
2890 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
2891 * longest flag, "relative", and to display at least a few node ids.
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002892 */
David Rientjes948927e2013-11-12 15:07:28 -08002893void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002894{
2895 char *p = buffer;
David Rientjes948927e2013-11-12 15:07:28 -08002896 nodemask_t nodes = NODE_MASK_NONE;
2897 unsigned short mode = MPOL_DEFAULT;
2898 unsigned short flags = 0;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002899
David Rientjes8790c71a2014-01-30 15:46:08 -08002900 if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002901 mode = pol->mode;
David Rientjes948927e2013-11-12 15:07:28 -08002902 flags = pol->flags;
2903 }
Lee Schermerhornbea904d2008-04-28 02:13:18 -07002904
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002905 switch (mode) {
2906 case MPOL_DEFAULT:
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002907 break;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002908 case MPOL_PREFERRED:
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002909 if (flags & MPOL_F_LOCAL)
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002910 mode = MPOL_LOCAL;
Lee Schermerhorn53f25562008-04-28 02:13:20 -07002911 else
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002912 node_set(pol->v.preferred_node, nodes);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002913 break;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002914 case MPOL_BIND:
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002915 case MPOL_INTERLEAVE:
Hugh Dickinsf2a07f42013-01-02 02:01:33 -08002916 nodes = pol->v.nodes;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002917 break;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002918 default:
David Rientjes948927e2013-11-12 15:07:28 -08002919 WARN_ON_ONCE(1);
2920 snprintf(p, maxlen, "unknown");
2921 return;
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002922 }
2923
David Rientjesb7a9f422013-11-21 14:32:06 -08002924 p += snprintf(p, maxlen, "%s", policy_modes[mode]);
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002925
Lee Schermerhornfc36b8d2008-04-28 02:13:21 -07002926 if (flags & MPOL_MODE_FLAGS) {
David Rientjes948927e2013-11-12 15:07:28 -08002927 p += snprintf(p, buffer + maxlen - p, "=");
David Rientjesf5b087b2008-04-28 02:12:27 -07002928
Lee Schermerhorn22919902008-04-28 02:13:22 -07002929 /*
2930 * Currently, the only defined flags are mutually exclusive
2931 */
David Rientjesf5b087b2008-04-28 02:12:27 -07002932 if (flags & MPOL_F_STATIC_NODES)
Lee Schermerhorn22919902008-04-28 02:13:22 -07002933 p += snprintf(p, buffer + maxlen - p, "static");
2934 else if (flags & MPOL_F_RELATIVE_NODES)
2935 p += snprintf(p, buffer + maxlen - p, "relative");
David Rientjesf5b087b2008-04-28 02:12:27 -07002936 }
2937
Tejun Heo9e763e02015-02-13 14:38:02 -08002938 if (!nodes_empty(nodes))
2939 p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
2940 nodemask_pr_args(&nodes));
Christoph Lameter1a75a6c2006-01-08 01:01:02 -08002941}