blob: f3418edb136be98dc943a096c0bf00b7f33b6220 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/mm/page_alloc.c
4 *
5 * Manages the free list, the system allocates free pages here.
6 * Note that kmalloc() lives in slab.c
7 *
8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
9 * Swap reorganised 29.12.95, Stephen Tweedie
10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
16 */
17
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/stddef.h>
19#include <linux/mm.h>
Arun KSca79b0c2018-12-28 00:34:29 -080020#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/swap.h>
22#include <linux/interrupt.h>
23#include <linux/pagemap.h>
KOSAKI Motohiro10ed2732008-03-04 14:28:32 -080024#include <linux/jiffies.h>
Yinghai Luedbe7d22010-08-25 13:39:16 -070025#include <linux/memblock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/compiler.h>
Randy Dunlap9f158332005-09-13 01:25:16 -070027#include <linux/kernel.h>
Andrey Ryabininb8c73fc2015-02-13 14:39:28 -080028#include <linux/kasan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/module.h>
30#include <linux/suspend.h>
31#include <linux/pagevec.h>
32#include <linux/blkdev.h>
33#include <linux/slab.h>
Dave Hansena238ab52011-05-24 17:12:16 -070034#include <linux/ratelimit.h>
David Rientjes5a3135c22007-10-16 23:25:53 -070035#include <linux/oom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <linux/topology.h>
37#include <linux/sysctl.h>
38#include <linux/cpu.h>
39#include <linux/cpuset.h>
Dave Hansenbdc8cb92005-10-29 18:16:53 -070040#include <linux/memory_hotplug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/nodemask.h>
42#include <linux/vmalloc.h>
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -070043#include <linux/vmstat.h>
Christoph Lameter4be38e32006-01-06 00:11:17 -080044#include <linux/mempolicy.h>
Dan Williams4b94ffd2016-01-15 16:56:22 -080045#include <linux/memremap.h>
Yasunori Goto68113782006-06-23 02:03:11 -070046#include <linux/stop_machine.h>
Dan Williams97500a42019-05-14 15:41:35 -070047#include <linux/random.h>
Mel Gormanc7132162006-09-27 01:49:43 -070048#include <linux/sort.h>
49#include <linux/pfn.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070050#include <linux/backing-dev.h>
Akinobu Mita933e3122006-12-08 02:39:45 -080051#include <linux/fault-inject.h>
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -070052#include <linux/page-isolation.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070053#include <linux/debugobjects.h>
Catalin Marinasdbb1f812009-06-11 13:23:19 +010054#include <linux/kmemleak.h>
Mel Gorman56de7262010-05-24 14:32:30 -070055#include <linux/compaction.h>
Mel Gorman0d3d0622009-09-21 17:02:44 -070056#include <trace/events/kmem.h>
Michal Hockod379f012017-02-22 15:42:00 -080057#include <trace/events/oom.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070058#include <linux/prefetch.h>
Lisa Du6e543d52013-09-11 14:22:36 -070059#include <linux/mm_inline.h>
Michal Nazarewicz041d3a82011-12-29 13:09:50 +010060#include <linux/migrate.h>
David Rientjes949f7ec2013-04-29 15:07:48 -070061#include <linux/hugetlb.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060062#include <linux/sched/rt.h>
Ingo Molnar5b3cc152017-02-02 20:43:54 +010063#include <linux/sched/mm.h>
Joonsoo Kim48c96a32014-12-12 16:56:01 -080064#include <linux/page_owner.h>
Mel Gorman0e1cc952015-06-30 14:57:27 -070065#include <linux/kthread.h>
Vladimir Davydov49491482016-07-26 15:24:24 -070066#include <linux/memcontrol.h>
Steven Rostedt (VMware)42c269c2017-03-03 16:15:39 -050067#include <linux/ftrace.h>
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +010068#include <linux/lockdep.h>
Chen Yu556b9692017-08-25 15:55:30 -070069#include <linux/nmi.h>
Johannes Weinereb414682018-10-26 15:06:27 -070070#include <linux/psi.h>
Daniel Jordane4443142020-06-03 15:59:51 -070071#include <linux/padata.h>
Vijay Balakrishna4aab2be2020-10-10 23:16:40 -070072#include <linux/khugepaged.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Jiang Liu7ee3d4e2013-07-03 15:03:41 -070074#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#include <asm/tlbflush.h>
Andrew Mortonac924c62006-05-15 09:43:59 -070076#include <asm/div64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070077#include "internal.h"
Dan Williamse900a912019-05-14 15:41:28 -070078#include "shuffle.h"
Alexander Duyck36e66c52020-04-06 20:04:56 -070079#include "page_reporting.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
David Hildenbrandf04a5d52020-10-15 20:09:20 -070081/* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
82typedef int __bitwise fpi_t;
83
84/* No special request */
85#define FPI_NONE ((__force fpi_t)0)
86
87/*
88 * Skip free page reporting notification for the (possibly merged) page.
89 * This does not hinder free page reporting from grabbing the page,
90 * reporting it and marking it "reported" - it only skips notifying
91 * the free page reporting infrastructure about a newly freed page. For
92 * example, used when temporarily pulling a page from a freelist and
93 * putting it back unmodified.
94 */
95#define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0))
96
David Hildenbrand47b6a24a22020-10-15 20:09:26 -070097/*
98 * Place the (possibly merged) page to the tail of the freelist. Will ignore
99 * page shuffling (relevant code - e.g., memory onlining - is expected to
100 * shuffle the whole zone).
101 *
102 * Note: No code should rely on this flag for correctness - it's purely
103 * to allow for optimizations when handing back either fresh pages
104 * (memory onlining) or untouched pages (page isolation, free page
105 * reporting).
106 */
107#define FPI_TO_TAIL ((__force fpi_t)BIT(1))
108
Cody P Schaferc8e251f2013-07-03 15:01:29 -0700109/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
110static DEFINE_MUTEX(pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -0700111#define MIN_PERCPU_PAGELIST_FRACTION (8)
Cody P Schaferc8e251f2013-07-03 15:01:29 -0700112
Lee Schermerhorn72812012010-05-26 14:44:56 -0700113#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
114DEFINE_PER_CPU(int, numa_node);
115EXPORT_PER_CPU_SYMBOL(numa_node);
116#endif
117
Kemi Wang45180852017-11-15 17:38:22 -0800118DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
119
Lee Schermerhorn7aac7892010-05-26 14:45:00 -0700120#ifdef CONFIG_HAVE_MEMORYLESS_NODES
121/*
122 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
123 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
124 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
125 * defined in <linux/topology.h>.
126 */
127DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
128EXPORT_PER_CPU_SYMBOL(_numa_mem_);
129#endif
130
Mel Gormanbd233f52017-02-24 14:56:56 -0800131/* work_structs for global per-cpu drains */
Wei Yangd9367bd2018-12-28 00:38:58 -0800132struct pcpu_drain {
133 struct zone *zone;
134 struct work_struct work;
135};
Jason Yan8b885f52020-04-10 14:32:32 -0700136static DEFINE_MUTEX(pcpu_drain_mutex);
137static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
Mel Gormanbd233f52017-02-24 14:56:56 -0800138
Emese Revfy38addce2016-06-20 20:41:19 +0200139#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
Kees Cook58bea412016-10-19 00:08:04 +0200140volatile unsigned long latent_entropy __latent_entropy;
Emese Revfy38addce2016-06-20 20:41:19 +0200141EXPORT_SYMBOL(latent_entropy);
142#endif
143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144/*
Christoph Lameter13808912007-10-16 01:25:27 -0700145 * Array of node states.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 */
Christoph Lameter13808912007-10-16 01:25:27 -0700147nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
148 [N_POSSIBLE] = NODE_MASK_ALL,
149 [N_ONLINE] = { { [0] = 1UL } },
150#ifndef CONFIG_NUMA
151 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
152#ifdef CONFIG_HIGHMEM
153 [N_HIGH_MEMORY] = { { [0] = 1UL } },
154#endif
Lai Jiangshan20b2f522012-12-12 13:52:00 -0800155 [N_MEMORY] = { { [0] = 1UL } },
Christoph Lameter13808912007-10-16 01:25:27 -0700156 [N_CPU] = { { [0] = 1UL } },
157#endif /* NUMA */
158};
159EXPORT_SYMBOL(node_states);
160
Arun KSca79b0c2018-12-28 00:34:29 -0800161atomic_long_t _totalram_pages __read_mostly;
162EXPORT_SYMBOL(_totalram_pages);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -0700163unsigned long totalreserve_pages __read_mostly;
Pintu Kumare48322a2014-12-18 16:17:15 -0800164unsigned long totalcma_pages __read_mostly;
Johannes Weinerab8fabd2012-01-10 15:07:42 -0800165
Hugh Dickins1b76b022012-05-11 01:00:07 -0700166int percpu_pagelist_fraction;
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +1000167gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
Alexander Potapenko64713842019-07-11 20:59:19 -0700168#ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON
169DEFINE_STATIC_KEY_TRUE(init_on_alloc);
170#else
171DEFINE_STATIC_KEY_FALSE(init_on_alloc);
172#endif
173EXPORT_SYMBOL(init_on_alloc);
174
175#ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON
176DEFINE_STATIC_KEY_TRUE(init_on_free);
177#else
178DEFINE_STATIC_KEY_FALSE(init_on_free);
179#endif
180EXPORT_SYMBOL(init_on_free);
181
182static int __init early_init_on_alloc(char *buf)
183{
184 int ret;
185 bool bool_result;
186
Alexander Potapenko64713842019-07-11 20:59:19 -0700187 ret = kstrtobool(buf, &bool_result);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700188 if (ret)
189 return ret;
Alexander Potapenko64713842019-07-11 20:59:19 -0700190 if (bool_result && page_poisoning_enabled())
191 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n");
192 if (bool_result)
193 static_branch_enable(&init_on_alloc);
194 else
195 static_branch_disable(&init_on_alloc);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700196 return 0;
Alexander Potapenko64713842019-07-11 20:59:19 -0700197}
198early_param("init_on_alloc", early_init_on_alloc);
199
200static int __init early_init_on_free(char *buf)
201{
202 int ret;
203 bool bool_result;
204
Alexander Potapenko64713842019-07-11 20:59:19 -0700205 ret = kstrtobool(buf, &bool_result);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700206 if (ret)
207 return ret;
Alexander Potapenko64713842019-07-11 20:59:19 -0700208 if (bool_result && page_poisoning_enabled())
209 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n");
210 if (bool_result)
211 static_branch_enable(&init_on_free);
212 else
213 static_branch_disable(&init_on_free);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700214 return 0;
Alexander Potapenko64713842019-07-11 20:59:19 -0700215}
216early_param("init_on_free", early_init_on_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -0700218/*
219 * A cached value of the page's pageblock's migratetype, used when the page is
220 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
221 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
222 * Also the migratetype set in the page does not necessarily match the pcplist
223 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
224 * other index - this ensures that it will be put on the correct CMA freelist.
225 */
226static inline int get_pcppage_migratetype(struct page *page)
227{
228 return page->index;
229}
230
231static inline void set_pcppage_migratetype(struct page *page, int migratetype)
232{
233 page->index = migratetype;
234}
235
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800236#ifdef CONFIG_PM_SLEEP
237/*
238 * The following functions are used by the suspend/hibernate code to temporarily
239 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
240 * while devices are suspended. To avoid races with the suspend/hibernate code,
Pingfan Liu55f25032018-07-31 16:51:32 +0800241 * they should always be called with system_transition_mutex held
242 * (gfp_allowed_mask also should only be modified with system_transition_mutex
243 * held, unless the suspend/hibernate code is guaranteed not to run in parallel
244 * with that modification).
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800245 */
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100246
247static gfp_t saved_gfp_mask;
248
249void pm_restore_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800250{
Pingfan Liu55f25032018-07-31 16:51:32 +0800251 WARN_ON(!mutex_is_locked(&system_transition_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100252 if (saved_gfp_mask) {
253 gfp_allowed_mask = saved_gfp_mask;
254 saved_gfp_mask = 0;
255 }
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800256}
257
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100258void pm_restrict_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800259{
Pingfan Liu55f25032018-07-31 16:51:32 +0800260 WARN_ON(!mutex_is_locked(&system_transition_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100261 WARN_ON(saved_gfp_mask);
262 saved_gfp_mask = gfp_allowed_mask;
Mel Gormand0164ad2015-11-06 16:28:21 -0800263 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800264}
Mel Gormanf90ac392012-01-10 15:07:15 -0800265
266bool pm_suspended_storage(void)
267{
Mel Gormand0164ad2015-11-06 16:28:21 -0800268 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
Mel Gormanf90ac392012-01-10 15:07:15 -0800269 return false;
270 return true;
271}
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800272#endif /* CONFIG_PM_SLEEP */
273
Mel Gormand9c23402007-10-16 01:26:01 -0700274#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800275unsigned int pageblock_order __read_mostly;
Mel Gormand9c23402007-10-16 01:26:01 -0700276#endif
277
David Hildenbrand7fef4312020-10-15 20:09:35 -0700278static void __free_pages_ok(struct page *page, unsigned int order,
279 fpi_t fpi_flags);
David Howellsa226f6c2006-01-06 00:11:08 -0800280
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281/*
282 * results with 256, 32 in the lowmem_reserve sysctl:
283 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
284 * 1G machine -> (16M dma, 784M normal, 224M high)
285 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
286 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
Yaowei Bai84109e12015-02-12 15:00:22 -0800287 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
Andi Kleena2f1b422005-11-05 17:25:53 +0100288 *
289 * TBD: should special case ZONE_DMA32 machines here - in those we normally
290 * don't need any ZONE_NORMAL reservation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 */
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700292int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800293#ifdef CONFIG_ZONE_DMA
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700294 [ZONE_DMA] = 256,
Christoph Lameter4b51d662007-02-10 01:43:10 -0800295#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700296#ifdef CONFIG_ZONE_DMA32
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700297 [ZONE_DMA32] = 256,
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700298#endif
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700299 [ZONE_NORMAL] = 32,
Christoph Lametere53ef382006-09-25 23:31:14 -0700300#ifdef CONFIG_HIGHMEM
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700301 [ZONE_HIGHMEM] = 0,
Christoph Lametere53ef382006-09-25 23:31:14 -0700302#endif
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700303 [ZONE_MOVABLE] = 0,
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700304};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305
Helge Deller15ad7cd2006-12-06 20:40:36 -0800306static char * const zone_names[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800307#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700308 "DMA",
Christoph Lameter4b51d662007-02-10 01:43:10 -0800309#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700310#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700311 "DMA32",
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700312#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700313 "Normal",
Christoph Lametere53ef382006-09-25 23:31:14 -0700314#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700315 "HighMem",
Christoph Lametere53ef382006-09-25 23:31:14 -0700316#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700317 "Movable",
Dan Williams033fbae2015-08-09 15:29:06 -0400318#ifdef CONFIG_ZONE_DEVICE
319 "Device",
320#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700321};
322
Alexey Dobriyanc999fbd2018-12-28 00:35:55 -0800323const char * const migratetype_names[MIGRATE_TYPES] = {
Vlastimil Babka60f30352016-03-15 14:56:08 -0700324 "Unmovable",
325 "Movable",
326 "Reclaimable",
327 "HighAtomic",
328#ifdef CONFIG_CMA
329 "CMA",
330#endif
331#ifdef CONFIG_MEMORY_ISOLATION
332 "Isolate",
333#endif
334};
335
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700336compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
337 [NULL_COMPOUND_DTOR] = NULL,
338 [COMPOUND_PAGE_DTOR] = free_compound_page,
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800339#ifdef CONFIG_HUGETLB_PAGE
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700340 [HUGETLB_PAGE_DTOR] = free_huge_page,
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800341#endif
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800342#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700343 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800344#endif
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800345};
346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347int min_free_kbytes = 1024;
Han Pingtian42aa83c2014-01-23 15:53:28 -0800348int user_min_free_kbytes = -1;
Mel Gorman24512228b2019-04-25 22:23:51 -0700349#ifdef CONFIG_DISCONTIGMEM
350/*
351 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges
352 * are not on separate NUMA nodes. Functionally this works but with
353 * watermark_boost_factor, it can reclaim prematurely as the ranges can be
354 * quite small. By default, do not boost watermarks on discontigmem as in
355 * many cases very high-order allocations like THP are likely to be
356 * unsupported and the premature reclaim offsets the advantage of long-term
357 * fragmentation avoidance.
358 */
359int watermark_boost_factor __read_mostly;
360#else
Mel Gorman1c308442018-12-28 00:35:52 -0800361int watermark_boost_factor __read_mostly = 15000;
Mel Gorman24512228b2019-04-25 22:23:51 -0700362#endif
Johannes Weiner795ae7a2016-03-17 14:19:14 -0700363int watermark_scale_factor = 10;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800365static unsigned long nr_kernel_pages __initdata;
366static unsigned long nr_all_pages __initdata;
367static unsigned long dma_reserve __initdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800369static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
370static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700371static unsigned long required_kernelcore __initdata;
David Rientjesa5c6d652018-04-05 16:23:09 -0700372static unsigned long required_kernelcore_percent __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700373static unsigned long required_movablecore __initdata;
David Rientjesa5c6d652018-04-05 16:23:09 -0700374static unsigned long required_movablecore_percent __initdata;
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800375static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700376static bool mirrored_kernelcore __meminitdata;
Mel Gormanc7132162006-09-27 01:49:43 -0700377
Tejun Heo0ee332c2011-12-08 10:22:09 -0800378/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
379int movable_zone;
380EXPORT_SYMBOL(movable_zone);
Mel Gormanc7132162006-09-27 01:49:43 -0700381
Miklos Szeredi418508c2007-05-23 13:57:55 -0700382#if MAX_NUMNODES > 1
Alexey Dobriyanb9726c22019-03-05 15:48:26 -0800383unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
Alexey Dobriyance0725f2019-03-05 15:48:29 -0800384unsigned int nr_online_nodes __read_mostly = 1;
Miklos Szeredi418508c2007-05-23 13:57:55 -0700385EXPORT_SYMBOL(nr_node_ids);
Christoph Lameter62bc62a2009-06-16 15:32:15 -0700386EXPORT_SYMBOL(nr_online_nodes);
Miklos Szeredi418508c2007-05-23 13:57:55 -0700387#endif
388
Mel Gorman9ef9acb2007-10-16 01:25:54 -0700389int page_group_by_mobility_disabled __read_mostly;
390
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700391#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Waiman Long3c0c12c2018-12-28 00:38:51 -0800392/*
393 * During boot we initialize deferred pages on-demand, as needed, but once
394 * page_alloc_init_late() has finished, the deferred pages are all initialized,
395 * and we can permanently disable that path.
396 */
397static DEFINE_STATIC_KEY_TRUE(deferred_pages);
398
399/*
400 * Calling kasan_free_pages() only after deferred memory initialization
401 * has completed. Poisoning pages during deferred memory init will greatly
402 * lengthen the process and cause problem in large memory systems as the
403 * deferred pages initialization is done with interrupt disabled.
404 *
405 * Assuming that there will be no reference to those newly initialized
406 * pages before they are ever allocated, this should have no effect on
407 * KASAN memory tracking as the poison will be properly inserted at page
408 * allocation time. The only corner case is when pages are allocated by
409 * on-demand allocation and then freed again before the deferred pages
410 * initialization is done, but this is not likely to happen.
411 */
412static inline void kasan_free_nondeferred_pages(struct page *page, int order)
413{
414 if (!static_branch_unlikely(&deferred_pages))
415 kasan_free_pages(page, order);
416}
417
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700418/* Returns true if the struct page for the pfn is uninitialised */
Mel Gorman0e1cc952015-06-30 14:57:27 -0700419static inline bool __meminit early_page_uninitialised(unsigned long pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700420{
Mel Gormanef70b6f2016-07-14 12:07:23 -0700421 int nid = early_pfn_to_nid(pfn);
422
423 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700424 return true;
425
426 return false;
427}
428
429/*
Pavel Tatashind3035be2018-10-26 15:09:37 -0700430 * Returns true when the remaining initialisation should be deferred until
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700431 * later in the boot cycle when it can be parallelised.
432 */
Pavel Tatashind3035be2018-10-26 15:09:37 -0700433static bool __meminit
434defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700435{
Pavel Tatashind3035be2018-10-26 15:09:37 -0700436 static unsigned long prev_end_pfn, nr_initialised;
437
438 /*
439 * prev_end_pfn static that contains the end of previous zone
440 * No need to protect because called very early in boot before smp_init.
441 */
442 if (prev_end_pfn != end_pfn) {
443 prev_end_pfn = end_pfn;
444 nr_initialised = 0;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700445 }
446
Pavel Tatashind3035be2018-10-26 15:09:37 -0700447 /* Always populate low zones for address-constrained allocations */
448 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
449 return false;
Wei Yang23b68cf2018-12-28 00:36:18 -0800450
Baoquan He98b57682020-12-29 15:14:37 -0800451 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
452 return true;
Wei Yang23b68cf2018-12-28 00:36:18 -0800453 /*
454 * We start only with one section of pages, more pages are added as
455 * needed until the rest of deferred pages are initialized.
456 */
Pavel Tatashind3035be2018-10-26 15:09:37 -0700457 nr_initialised++;
Wei Yang23b68cf2018-12-28 00:36:18 -0800458 if ((nr_initialised > PAGES_PER_SECTION) &&
Pavel Tatashind3035be2018-10-26 15:09:37 -0700459 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
460 NODE_DATA(nid)->first_deferred_pfn = pfn;
461 return true;
462 }
463 return false;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700464}
465#else
Waiman Long3c0c12c2018-12-28 00:38:51 -0800466#define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o)
467
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700468static inline bool early_page_uninitialised(unsigned long pfn)
469{
470 return false;
471}
472
Pavel Tatashind3035be2018-10-26 15:09:37 -0700473static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700474{
Pavel Tatashind3035be2018-10-26 15:09:37 -0700475 return false;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700476}
477#endif
478
Mel Gorman0b423ca2016-05-19 17:14:27 -0700479/* Return a pointer to the bitmap storing bits affecting a block of pages */
480static inline unsigned long *get_pageblock_bitmap(struct page *page,
481 unsigned long pfn)
482{
483#ifdef CONFIG_SPARSEMEM
Dan Williamsf1eca352019-07-18 15:57:57 -0700484 return section_to_usemap(__pfn_to_section(pfn));
Mel Gorman0b423ca2016-05-19 17:14:27 -0700485#else
486 return page_zone(page)->pageblock_flags;
487#endif /* CONFIG_SPARSEMEM */
488}
489
490static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
491{
492#ifdef CONFIG_SPARSEMEM
493 pfn &= (PAGES_PER_SECTION-1);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700494#else
495 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700496#endif /* CONFIG_SPARSEMEM */
Wei Yang399b7952020-08-06 23:25:44 -0700497 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700498}
499
500/**
501 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
502 * @page: The page within the block of interest
503 * @pfn: The target page frame number
Mel Gorman0b423ca2016-05-19 17:14:27 -0700504 * @mask: mask of bits that the caller is interested in
505 *
506 * Return: pageblock_bits flags
507 */
Wei Yang535b81e2020-08-06 23:25:51 -0700508static __always_inline
509unsigned long __get_pfnblock_flags_mask(struct page *page,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700510 unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700511 unsigned long mask)
512{
513 unsigned long *bitmap;
514 unsigned long bitidx, word_bitidx;
515 unsigned long word;
516
517 bitmap = get_pageblock_bitmap(page, pfn);
518 bitidx = pfn_to_bitidx(page, pfn);
519 word_bitidx = bitidx / BITS_PER_LONG;
520 bitidx &= (BITS_PER_LONG-1);
521
522 word = bitmap[word_bitidx];
Wei Yangd93d5ab2020-08-06 23:25:48 -0700523 return (word >> bitidx) & mask;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700524}
525
526unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700527 unsigned long mask)
528{
Wei Yang535b81e2020-08-06 23:25:51 -0700529 return __get_pfnblock_flags_mask(page, pfn, mask);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700530}
531
532static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
533{
Wei Yang535b81e2020-08-06 23:25:51 -0700534 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700535}
536
537/**
538 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
539 * @page: The page within the block of interest
540 * @flags: The flags to set
541 * @pfn: The target page frame number
Mel Gorman0b423ca2016-05-19 17:14:27 -0700542 * @mask: mask of bits that the caller is interested in
543 */
544void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
545 unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700546 unsigned long mask)
547{
548 unsigned long *bitmap;
549 unsigned long bitidx, word_bitidx;
550 unsigned long old_word, word;
551
552 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
Pingfan Liu125b8602018-12-28 00:38:43 -0800553 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
Mel Gorman0b423ca2016-05-19 17:14:27 -0700554
555 bitmap = get_pageblock_bitmap(page, pfn);
556 bitidx = pfn_to_bitidx(page, pfn);
557 word_bitidx = bitidx / BITS_PER_LONG;
558 bitidx &= (BITS_PER_LONG-1);
559
560 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
561
Wei Yangd93d5ab2020-08-06 23:25:48 -0700562 mask <<= bitidx;
563 flags <<= bitidx;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700564
565 word = READ_ONCE(bitmap[word_bitidx]);
566 for (;;) {
567 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
568 if (word == old_word)
569 break;
570 word = old_word;
571 }
572}
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700573
Minchan Kimee6f5092012-07-31 16:43:50 -0700574void set_pageblock_migratetype(struct page *page, int migratetype)
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700575{
KOSAKI Motohiro5d0f3f72013-11-12 15:08:18 -0800576 if (unlikely(page_group_by_mobility_disabled &&
577 migratetype < MIGRATE_PCPTYPES))
Mel Gorman49255c62009-06-16 15:31:58 -0700578 migratetype = MIGRATE_UNMOVABLE;
579
Wei Yangd93d5ab2020-08-06 23:25:48 -0700580 set_pfnblock_flags_mask(page, (unsigned long)migratetype,
Wei Yang535b81e2020-08-06 23:25:51 -0700581 page_to_pfn(page), MIGRATETYPE_MASK);
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700582}
583
Nick Piggin13e74442006-01-06 00:10:58 -0800584#ifdef CONFIG_DEBUG_VM
Dave Hansenc6a57e12005-10-29 18:16:52 -0700585static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586{
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700587 int ret = 0;
588 unsigned seq;
589 unsigned long pfn = page_to_pfn(page);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800590 unsigned long sp, start_pfn;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700591
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700592 do {
593 seq = zone_span_seqbegin(zone);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800594 start_pfn = zone->zone_start_pfn;
595 sp = zone->spanned_pages;
Cody P Schafer108bcc92013-02-22 16:35:23 -0800596 if (!zone_spans_pfn(zone, pfn))
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700597 ret = 1;
598 } while (zone_span_seqretry(zone, seq));
599
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800600 if (ret)
Dave Hansen613813e2014-06-04 16:07:27 -0700601 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
602 pfn, zone_to_nid(zone), zone->name,
603 start_pfn, start_pfn + sp);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800604
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700605 return ret;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700606}
607
608static int page_is_consistent(struct zone *zone, struct page *page)
609{
Andy Whitcroft14e07292007-05-06 14:49:14 -0700610 if (!pfn_valid_within(page_to_pfn(page)))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700611 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 if (zone != page_zone(page))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700613 return 0;
614
615 return 1;
616}
617/*
618 * Temporary debugging check for pages not lying within a given zone.
619 */
Matthias Kaehlcked73d3c9f2017-07-06 15:39:23 -0700620static int __maybe_unused bad_range(struct zone *zone, struct page *page)
Dave Hansenc6a57e12005-10-29 18:16:52 -0700621{
622 if (page_outside_zone_boundaries(zone, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 return 1;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700624 if (!page_is_consistent(zone, page))
625 return 1;
626
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 return 0;
628}
Nick Piggin13e74442006-01-06 00:10:58 -0800629#else
Matthias Kaehlcked73d3c9f2017-07-06 15:39:23 -0700630static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
Nick Piggin13e74442006-01-06 00:10:58 -0800631{
632 return 0;
633}
634#endif
635
Wei Yang82a32412020-06-03 15:58:29 -0700636static void bad_page(struct page *page, const char *reason)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637{
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800638 static unsigned long resume;
639 static unsigned long nr_shown;
640 static unsigned long nr_unshown;
641
642 /*
643 * Allow a burst of 60 reports, then keep quiet for that minute;
644 * or allow a steady drip of one report per second.
645 */
646 if (nr_shown == 60) {
647 if (time_before(jiffies, resume)) {
648 nr_unshown++;
649 goto out;
650 }
651 if (nr_unshown) {
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700652 pr_alert(
Hugh Dickins1e9e6362009-01-06 14:40:13 -0800653 "BUG: Bad page state: %lu messages suppressed\n",
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800654 nr_unshown);
655 nr_unshown = 0;
656 }
657 nr_shown = 0;
658 }
659 if (nr_shown++ == 0)
660 resume = jiffies + 60 * HZ;
661
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700662 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
Hugh Dickins3dc14742009-01-06 14:40:08 -0800663 current->comm, page_to_pfn(page));
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700664 __dump_page(page, reason);
Vlastimil Babka4e462112016-03-15 14:56:21 -0700665 dump_page_owner(page);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700666
Dave Jones4f318882011-10-31 17:07:24 -0700667 print_modules();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 dump_stack();
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800669out:
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800670 /* Leave bad fields for debug, except PageBuddy could make trouble */
Mel Gorman22b751c2013-02-22 16:34:59 -0800671 page_mapcount_reset(page); /* remove PageBuddy */
Rusty Russell373d4d02013-01-21 17:17:39 +1030672 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673}
674
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675/*
676 * Higher-order pages are called "compound pages". They are structured thusly:
677 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800678 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800680 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
681 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800683 * The first tail page's ->compound_dtor holds the offset in array of compound
684 * page destructors. See compound_page_dtors.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800686 * The first tail page's ->compound_order holds the order of allocation.
Hugh Dickins41d78ba2006-02-14 13:52:58 -0800687 * This usage means that zero-order pages may not be compound.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 */
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800689
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800690void free_compound_page(struct page *page)
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800691{
Yang Shi7ae88532019-09-23 15:38:09 -0700692 mem_cgroup_uncharge(page);
David Hildenbrand7fef4312020-10-15 20:09:35 -0700693 __free_pages_ok(page, compound_order(page), FPI_NONE);
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800694}
695
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800696void prep_compound_page(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697{
698 int i;
699 int nr_pages = 1 << order;
700
Christoph Lameter6d777952007-05-06 14:49:40 -0700701 __SetPageHead(page);
Andy Whitcroft18229df2008-11-06 12:53:27 -0800702 for (i = 1; i < nr_pages; i++) {
703 struct page *p = page + i;
Youquan Song58a84aa2011-12-08 14:34:18 -0800704 set_page_count(p, 0);
Kirill A. Shutemov1c290f62016-01-15 16:52:07 -0800705 p->mapping = TAIL_MAPPING;
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800706 set_compound_head(p, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 }
Matthew Wilcox (Oracle)1378a5e2020-08-14 17:30:23 -0700708
709 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
710 set_compound_order(page, order);
Kirill A. Shutemov53f92632016-01-15 16:53:42 -0800711 atomic_set(compound_mapcount_ptr(page), -1);
John Hubbard47e29d32020-04-01 21:05:33 -0700712 if (hpage_pincount_available(page))
713 atomic_set(compound_pincount_ptr(page), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714}
715
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800716#ifdef CONFIG_DEBUG_PAGEALLOC
717unsigned int _debug_guardpage_minorder;
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700718
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800719bool _debug_pagealloc_enabled_early __read_mostly
720 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
721EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700722DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
Joonsoo Kim505f6d22016-03-17 14:17:56 -0700723EXPORT_SYMBOL(_debug_pagealloc_enabled);
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700724
725DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800726
Joonsoo Kim031bc572014-12-12 16:55:52 -0800727static int __init early_debug_pagealloc(char *buf)
728{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800729 return kstrtobool(buf, &_debug_pagealloc_enabled_early);
Joonsoo Kim031bc572014-12-12 16:55:52 -0800730}
731early_param("debug_pagealloc", early_debug_pagealloc);
732
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800733void init_debug_pagealloc(void)
Joonsoo Kime30825f2014-12-12 16:55:49 -0800734{
Joonsoo Kim031bc572014-12-12 16:55:52 -0800735 if (!debug_pagealloc_enabled())
736 return;
737
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800738 static_branch_enable(&_debug_pagealloc_enabled);
739
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700740 if (!debug_guardpage_minorder())
741 return;
742
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700743 static_branch_enable(&_debug_guardpage_enabled);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800744}
745
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800746static int __init debug_guardpage_minorder_setup(char *buf)
747{
748 unsigned long res;
749
750 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
Joe Perches11705322016-03-17 14:19:50 -0700751 pr_err("Bad debug_guardpage_minorder value\n");
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800752 return 0;
753 }
754 _debug_guardpage_minorder = res;
Joe Perches11705322016-03-17 14:19:50 -0700755 pr_info("Setting debug_guardpage_minorder to %lu\n", res);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800756 return 0;
757}
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700758early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800759
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700760static inline bool set_page_guard(struct zone *zone, struct page *page,
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800761 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800762{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800763 if (!debug_guardpage_enabled())
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700764 return false;
765
766 if (order >= debug_guardpage_minorder())
767 return false;
Joonsoo Kime30825f2014-12-12 16:55:49 -0800768
Vlastimil Babka3972f6b2019-07-11 20:55:13 -0700769 __SetPageGuard(page);
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800770 INIT_LIST_HEAD(&page->lru);
771 set_page_private(page, order);
772 /* Guard pages are not available for any usage */
773 __mod_zone_freepage_state(zone, -(1 << order), migratetype);
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700774
775 return true;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800776}
777
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800778static inline void clear_page_guard(struct zone *zone, struct page *page,
779 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800780{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800781 if (!debug_guardpage_enabled())
782 return;
783
Vlastimil Babka3972f6b2019-07-11 20:55:13 -0700784 __ClearPageGuard(page);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800785
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800786 set_page_private(page, 0);
787 if (!is_migrate_isolate(migratetype))
788 __mod_zone_freepage_state(zone, (1 << order), migratetype);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800789}
790#else
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700791static inline bool set_page_guard(struct zone *zone, struct page *page,
792 unsigned int order, int migratetype) { return false; }
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800793static inline void clear_page_guard(struct zone *zone, struct page *page,
794 unsigned int order, int migratetype) {}
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800795#endif
796
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700797static inline void set_buddy_order(struct page *page, unsigned int order)
Andrew Morton6aa3001b22006-04-18 22:20:52 -0700798{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700799 set_page_private(page, order);
Nick Piggin676165a2006-04-10 11:21:48 +1000800 __SetPageBuddy(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801}
802
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 * This function checks whether a page is free && is the buddy
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700805 * we can coalesce a page and its buddy if
Vlastimil Babka13ad59d2017-02-22 15:41:51 -0800806 * (a) the buddy is not in a hole (check before calling!) &&
Nick Piggin676165a2006-04-10 11:21:48 +1000807 * (b) the buddy is in the buddy system &&
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700808 * (c) a page and its buddy have the same order &&
809 * (d) a page and its buddy are in the same zone.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 *
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700811 * For recording whether a page is in the buddy system, we set PageBuddy.
812 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
Nick Piggin676165a2006-04-10 11:21:48 +1000813 *
814 * For recording page's order, we use page_private(page).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 */
chenqiwufe925c02020-04-01 21:09:56 -0700816static inline bool page_is_buddy(struct page *page, struct page *buddy,
Mel Gorman7aeb09f2014-06-04 16:10:21 -0700817 unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818{
chenqiwufe925c02020-04-01 21:09:56 -0700819 if (!page_is_guard(buddy) && !PageBuddy(buddy))
820 return false;
Mel Gormand34c5fa2014-06-04 16:10:10 -0700821
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700822 if (buddy_order(buddy) != order)
chenqiwufe925c02020-04-01 21:09:56 -0700823 return false;
Weijie Yang4c5018c2015-02-10 14:11:39 -0800824
chenqiwufe925c02020-04-01 21:09:56 -0700825 /*
826 * zone check is done late to avoid uselessly calculating
827 * zone/node ids for pages that could never merge.
828 */
829 if (page_zone_id(page) != page_zone_id(buddy))
830 return false;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800831
chenqiwufe925c02020-04-01 21:09:56 -0700832 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
Mel Gormand34c5fa2014-06-04 16:10:10 -0700833
chenqiwufe925c02020-04-01 21:09:56 -0700834 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835}
836
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800837#ifdef CONFIG_COMPACTION
838static inline struct capture_control *task_capc(struct zone *zone)
839{
840 struct capture_control *capc = current->capture_control;
841
Vlastimil Babkadeba0482020-08-06 23:25:16 -0700842 return unlikely(capc) &&
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800843 !(current->flags & PF_KTHREAD) &&
844 !capc->page &&
Vlastimil Babkadeba0482020-08-06 23:25:16 -0700845 capc->cc->zone == zone ? capc : NULL;
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800846}
847
848static inline bool
849compaction_capture(struct capture_control *capc, struct page *page,
850 int order, int migratetype)
851{
852 if (!capc || order != capc->cc->order)
853 return false;
854
855 /* Do not accidentally pollute CMA or isolated regions*/
856 if (is_migrate_cma(migratetype) ||
857 is_migrate_isolate(migratetype))
858 return false;
859
860 /*
861 * Do not let lower order allocations polluate a movable pageblock.
862 * This might let an unmovable request use a reclaimable pageblock
863 * and vice-versa but no more than normal fallback logic which can
864 * have trouble finding a high-order free page.
865 */
866 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
867 return false;
868
869 capc->page = page;
870 return true;
871}
872
873#else
874static inline struct capture_control *task_capc(struct zone *zone)
875{
876 return NULL;
877}
878
879static inline bool
880compaction_capture(struct capture_control *capc, struct page *page,
881 int order, int migratetype)
882{
883 return false;
884}
885#endif /* CONFIG_COMPACTION */
886
Alexander Duyck6ab01362020-04-06 20:04:49 -0700887/* Used for pages not on another list */
888static inline void add_to_free_list(struct page *page, struct zone *zone,
889 unsigned int order, int migratetype)
890{
891 struct free_area *area = &zone->free_area[order];
892
893 list_add(&page->lru, &area->free_list[migratetype]);
894 area->nr_free++;
895}
896
897/* Used for pages not on another list */
898static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
899 unsigned int order, int migratetype)
900{
901 struct free_area *area = &zone->free_area[order];
902
903 list_add_tail(&page->lru, &area->free_list[migratetype]);
904 area->nr_free++;
905}
906
David Hildenbrand293ffa52020-10-15 20:09:30 -0700907/*
908 * Used for pages which are on another list. Move the pages to the tail
909 * of the list - so the moved pages won't immediately be considered for
910 * allocation again (e.g., optimization for memory onlining).
911 */
Alexander Duyck6ab01362020-04-06 20:04:49 -0700912static inline void move_to_free_list(struct page *page, struct zone *zone,
913 unsigned int order, int migratetype)
914{
915 struct free_area *area = &zone->free_area[order];
916
David Hildenbrand293ffa52020-10-15 20:09:30 -0700917 list_move_tail(&page->lru, &area->free_list[migratetype]);
Alexander Duyck6ab01362020-04-06 20:04:49 -0700918}
919
920static inline void del_page_from_free_list(struct page *page, struct zone *zone,
921 unsigned int order)
922{
Alexander Duyck36e66c52020-04-06 20:04:56 -0700923 /* clear reported state and update reported page count */
924 if (page_reported(page))
925 __ClearPageReported(page);
926
Alexander Duyck6ab01362020-04-06 20:04:49 -0700927 list_del(&page->lru);
928 __ClearPageBuddy(page);
929 set_page_private(page, 0);
930 zone->free_area[order].nr_free--;
931}
932
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933/*
Alexander Duycka2129f22020-04-06 20:04:45 -0700934 * If this is not the largest possible page, check if the buddy
935 * of the next-highest order is free. If it is, it's possible
936 * that pages are being freed that will coalesce soon. In case,
937 * that is happening, add the free page to the tail of the list
938 * so it's less likely to be used soon and more likely to be merged
939 * as a higher order page
940 */
941static inline bool
942buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
943 struct page *page, unsigned int order)
944{
945 struct page *higher_page, *higher_buddy;
946 unsigned long combined_pfn;
947
948 if (order >= MAX_ORDER - 2)
949 return false;
950
951 if (!pfn_valid_within(buddy_pfn))
952 return false;
953
954 combined_pfn = buddy_pfn & pfn;
955 higher_page = page + (combined_pfn - pfn);
956 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
957 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
958
959 return pfn_valid_within(buddy_pfn) &&
960 page_is_buddy(higher_page, higher_buddy, order + 1);
961}
962
963/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 * Freeing function for a buddy system allocator.
965 *
966 * The concept of a buddy system is to maintain direct-mapped table
967 * (containing bit values) for memory blocks of various "orders".
968 * The bottom level table contains the map for the smallest allocatable
969 * units of memory (here, pages), and each level above it describes
970 * pairs of units from the levels below, hence, "buddies".
971 * At a high level, all that happens here is marking the table entry
972 * at the bottom level available, and propagating the changes upward
973 * as necessary, plus some accounting needed to play nicely with other
974 * parts of the VM system.
975 * At each level, we keep a list of pages, which are heads of continuous
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700976 * free pages of length of (1 << order) and marked with PageBuddy.
977 * Page's order is recorded in page_private(page) field.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 * So when we are allocating or freeing one, we can derive the state of the
Michal Nazarewicz5f63b722012-01-11 15:16:11 +0100979 * other. That is, if we allocate a small block, and both were
980 * free, the remainder of the region must be split into blocks.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 * If a block is freed, and its buddy is also free, then this
Michal Nazarewicz5f63b722012-01-11 15:16:11 +0100982 * triggers coalescing into a block of larger size.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +0100984 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 */
986
Nick Piggin48db57f2006-01-08 01:00:42 -0800987static inline void __free_one_page(struct page *page,
Mel Gormandc4b0ca2014-06-04 16:10:17 -0700988 unsigned long pfn,
Mel Gormaned0ae212009-06-16 15:32:07 -0700989 struct zone *zone, unsigned int order,
David Hildenbrandf04a5d52020-10-15 20:09:20 -0700990 int migratetype, fpi_t fpi_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991{
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800992 struct capture_control *capc = task_capc(zone);
Kees Cook3f649ab2020-06-03 13:09:38 -0700993 unsigned long buddy_pfn;
Alexander Duycka2129f22020-04-06 20:04:45 -0700994 unsigned long combined_pfn;
Alexander Duycka2129f22020-04-06 20:04:45 -0700995 unsigned int max_order;
996 struct page *buddy;
997 bool to_tail;
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700998
Muchun Songc42259922020-12-14 19:11:25 -0800999 max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000
Cody P Schaferd29bb972013-02-22 16:35:25 -08001001 VM_BUG_ON(!zone_is_initialized(zone));
Kirill A. Shutemov6e9f0d52015-02-11 15:25:50 -08001002 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003
Mel Gormaned0ae212009-06-16 15:32:07 -07001004 VM_BUG_ON(migratetype == -1);
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001005 if (likely(!is_migrate_isolate(migratetype)))
Joonsoo Kim8f82b552014-11-13 15:19:18 -08001006 __mod_zone_freepage_state(zone, 1 << order, migratetype);
Mel Gormaned0ae212009-06-16 15:32:07 -07001007
Vlastimil Babka76741e72017-02-22 15:41:48 -08001008 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
Sasha Levin309381fea2014-01-23 15:52:54 -08001009 VM_BUG_ON_PAGE(bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001011continue_merging:
Muchun Songc42259922020-12-14 19:11:25 -08001012 while (order < max_order) {
Mel Gorman5e1f0f02019-03-05 15:45:41 -08001013 if (compaction_capture(capc, page, order, migratetype)) {
1014 __mod_zone_freepage_state(zone, -(1 << order),
1015 migratetype);
1016 return;
1017 }
Vlastimil Babka76741e72017-02-22 15:41:48 -08001018 buddy_pfn = __find_buddy_pfn(pfn, order);
1019 buddy = page + (buddy_pfn - pfn);
Vlastimil Babka13ad59d2017-02-22 15:41:51 -08001020
1021 if (!pfn_valid_within(buddy_pfn))
1022 goto done_merging;
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -07001023 if (!page_is_buddy(page, buddy, order))
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001024 goto done_merging;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08001025 /*
1026 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
1027 * merge with it and move up one order.
1028 */
Dan Williamsb03641a2019-05-14 15:41:32 -07001029 if (page_is_guard(buddy))
Joonsoo Kim2847cf92014-12-12 16:55:01 -08001030 clear_page_guard(zone, buddy, order, migratetype);
Dan Williamsb03641a2019-05-14 15:41:32 -07001031 else
Alexander Duyck6ab01362020-04-06 20:04:49 -07001032 del_page_from_free_list(buddy, zone, order);
Vlastimil Babka76741e72017-02-22 15:41:48 -08001033 combined_pfn = buddy_pfn & pfn;
1034 page = page + (combined_pfn - pfn);
1035 pfn = combined_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 order++;
1037 }
Muchun Songc42259922020-12-14 19:11:25 -08001038 if (order < MAX_ORDER - 1) {
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001039 /* If we are here, it means order is >= pageblock_order.
1040 * We want to prevent merge between freepages on isolate
1041 * pageblock and normal pageblock. Without this, pageblock
1042 * isolation could cause incorrect freepage or CMA accounting.
1043 *
1044 * We don't want to hit this code for the more frequent
1045 * low-order merging.
1046 */
1047 if (unlikely(has_isolate_pageblock(zone))) {
1048 int buddy_mt;
1049
Vlastimil Babka76741e72017-02-22 15:41:48 -08001050 buddy_pfn = __find_buddy_pfn(pfn, order);
1051 buddy = page + (buddy_pfn - pfn);
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001052 buddy_mt = get_pageblock_migratetype(buddy);
1053
1054 if (migratetype != buddy_mt
1055 && (is_migrate_isolate(migratetype) ||
1056 is_migrate_isolate(buddy_mt)))
1057 goto done_merging;
1058 }
Muchun Songc42259922020-12-14 19:11:25 -08001059 max_order = order + 1;
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001060 goto continue_merging;
1061 }
1062
1063done_merging:
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07001064 set_buddy_order(page, order);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -07001065
David Hildenbrand47b6a24a22020-10-15 20:09:26 -07001066 if (fpi_flags & FPI_TO_TAIL)
1067 to_tail = true;
1068 else if (is_shuffle_order(order))
Alexander Duycka2129f22020-04-06 20:04:45 -07001069 to_tail = shuffle_pick_tail();
Dan Williams97500a42019-05-14 15:41:35 -07001070 else
Alexander Duycka2129f22020-04-06 20:04:45 -07001071 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
Dan Williams97500a42019-05-14 15:41:35 -07001072
Alexander Duycka2129f22020-04-06 20:04:45 -07001073 if (to_tail)
Alexander Duyck6ab01362020-04-06 20:04:49 -07001074 add_to_free_list_tail(page, zone, order, migratetype);
Alexander Duycka2129f22020-04-06 20:04:45 -07001075 else
Alexander Duyck6ab01362020-04-06 20:04:49 -07001076 add_to_free_list(page, zone, order, migratetype);
Alexander Duyck36e66c52020-04-06 20:04:56 -07001077
1078 /* Notify page reporting subsystem of freed page */
David Hildenbrandf04a5d52020-10-15 20:09:20 -07001079 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
Alexander Duyck36e66c52020-04-06 20:04:56 -07001080 page_reporting_notify_free(order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081}
1082
Mel Gorman7bfec6f2016-05-19 17:14:15 -07001083/*
1084 * A bad page could be due to a number of fields. Instead of multiple branches,
1085 * try and check multiple fields with one check. The caller must do a detailed
1086 * check if necessary.
1087 */
1088static inline bool page_expected_state(struct page *page,
1089 unsigned long check_flags)
1090{
1091 if (unlikely(atomic_read(&page->_mapcount) != -1))
1092 return false;
1093
1094 if (unlikely((unsigned long)page->mapping |
1095 page_ref_count(page) |
1096#ifdef CONFIG_MEMCG
1097 (unsigned long)page->mem_cgroup |
1098#endif
1099 (page->flags & check_flags)))
1100 return false;
1101
1102 return true;
1103}
1104
Wei Yang58b7f112020-06-03 15:58:39 -07001105static const char *page_bad_reason(struct page *page, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106{
Wei Yang82a32412020-06-03 15:58:29 -07001107 const char *bad_reason = NULL;
Dave Hansenf0b791a2014-01-23 15:52:49 -08001108
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08001109 if (unlikely(atomic_read(&page->_mapcount) != -1))
Dave Hansenf0b791a2014-01-23 15:52:49 -08001110 bad_reason = "nonzero mapcount";
1111 if (unlikely(page->mapping != NULL))
1112 bad_reason = "non-NULL mapping";
Joonsoo Kimfe896d12016-03-17 14:19:26 -07001113 if (unlikely(page_ref_count(page) != 0))
Joonsoo Kim0139aa72016-05-19 17:10:49 -07001114 bad_reason = "nonzero _refcount";
Wei Yang58b7f112020-06-03 15:58:39 -07001115 if (unlikely(page->flags & flags)) {
1116 if (flags == PAGE_FLAGS_CHECK_AT_PREP)
1117 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
1118 else
1119 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
Dave Hansenf0b791a2014-01-23 15:52:49 -08001120 }
Johannes Weiner9edad6e2014-12-10 15:44:58 -08001121#ifdef CONFIG_MEMCG
1122 if (unlikely(page->mem_cgroup))
1123 bad_reason = "page still charged to cgroup";
1124#endif
Wei Yang58b7f112020-06-03 15:58:39 -07001125 return bad_reason;
Mel Gormanbb552ac2016-05-19 17:14:18 -07001126}
1127
Wei Yang58b7f112020-06-03 15:58:39 -07001128static void check_free_page_bad(struct page *page)
1129{
1130 bad_page(page,
1131 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
Mel Gormanbb552ac2016-05-19 17:14:18 -07001132}
1133
Wei Yang534fe5e2020-06-03 15:58:36 -07001134static inline int check_free_page(struct page *page)
Mel Gormanbb552ac2016-05-19 17:14:18 -07001135{
Mel Gormanda838d42016-05-19 17:14:21 -07001136 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
Mel Gormanbb552ac2016-05-19 17:14:18 -07001137 return 0;
Mel Gormanbb552ac2016-05-19 17:14:18 -07001138
1139 /* Something has gone sideways, find it */
Wei Yang0d0c48a2020-06-03 15:58:33 -07001140 check_free_page_bad(page);
Mel Gorman7bfec6f2016-05-19 17:14:15 -07001141 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142}
1143
Mel Gorman4db75482016-05-19 17:14:32 -07001144static int free_tail_pages_check(struct page *head_page, struct page *page)
1145{
1146 int ret = 1;
1147
1148 /*
1149 * We rely page->lru.next never has bit 0 set, unless the page
1150 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
1151 */
1152 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1153
1154 if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1155 ret = 0;
1156 goto out;
1157 }
1158 switch (page - head_page) {
1159 case 1:
Matthew Wilcox4da19842018-06-07 17:08:50 -07001160 /* the first tail page: ->mapping may be compound_mapcount() */
Mel Gorman4db75482016-05-19 17:14:32 -07001161 if (unlikely(compound_mapcount(page))) {
Wei Yang82a32412020-06-03 15:58:29 -07001162 bad_page(page, "nonzero compound_mapcount");
Mel Gorman4db75482016-05-19 17:14:32 -07001163 goto out;
1164 }
1165 break;
1166 case 2:
1167 /*
1168 * the second tail page: ->mapping is
Matthew Wilcoxfa3015b2018-06-07 17:08:42 -07001169 * deferred_list.next -- ignore value.
Mel Gorman4db75482016-05-19 17:14:32 -07001170 */
1171 break;
1172 default:
1173 if (page->mapping != TAIL_MAPPING) {
Wei Yang82a32412020-06-03 15:58:29 -07001174 bad_page(page, "corrupted mapping in tail page");
Mel Gorman4db75482016-05-19 17:14:32 -07001175 goto out;
1176 }
1177 break;
1178 }
1179 if (unlikely(!PageTail(page))) {
Wei Yang82a32412020-06-03 15:58:29 -07001180 bad_page(page, "PageTail not set");
Mel Gorman4db75482016-05-19 17:14:32 -07001181 goto out;
1182 }
1183 if (unlikely(compound_head(page) != head_page)) {
Wei Yang82a32412020-06-03 15:58:29 -07001184 bad_page(page, "compound_head not consistent");
Mel Gorman4db75482016-05-19 17:14:32 -07001185 goto out;
1186 }
1187 ret = 0;
1188out:
1189 page->mapping = NULL;
1190 clear_compound_head(page);
1191 return ret;
1192}
1193
Alexander Potapenko64713842019-07-11 20:59:19 -07001194static void kernel_init_free_pages(struct page *page, int numpages)
1195{
1196 int i;
1197
Qian Cai9e15afa2020-08-06 23:25:54 -07001198 /* s390's use of memset() could override KASAN redzones. */
1199 kasan_disable_current();
Alexander Potapenko64713842019-07-11 20:59:19 -07001200 for (i = 0; i < numpages; i++)
1201 clear_highpage(page + i);
Qian Cai9e15afa2020-08-06 23:25:54 -07001202 kasan_enable_current();
Alexander Potapenko64713842019-07-11 20:59:19 -07001203}
1204
Mel Gormane2769db2016-05-19 17:14:38 -07001205static __always_inline bool free_pages_prepare(struct page *page,
1206 unsigned int order, bool check_free)
1207{
1208 int bad = 0;
1209
1210 VM_BUG_ON_PAGE(PageTail(page), page);
1211
1212 trace_mm_page_free(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001213
Oscar Salvador79f5f8f2020-10-15 20:07:09 -07001214 if (unlikely(PageHWPoison(page)) && !order) {
1215 /*
1216 * Do not let hwpoison pages hit pcplists/buddy
1217 * Untie memcg state and reset page's owner
1218 */
1219 if (memcg_kmem_enabled() && PageKmemcg(page))
1220 __memcg_kmem_uncharge_page(page, order);
1221 reset_page_owner(page, order);
1222 return false;
1223 }
1224
Mel Gormane2769db2016-05-19 17:14:38 -07001225 /*
1226 * Check tail pages before head page information is cleared to
1227 * avoid checking PageCompound for order-0 pages.
1228 */
1229 if (unlikely(order)) {
1230 bool compound = PageCompound(page);
1231 int i;
1232
1233 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1234
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001235 if (compound)
1236 ClearPageDoubleMap(page);
Mel Gormane2769db2016-05-19 17:14:38 -07001237 for (i = 1; i < (1 << order); i++) {
1238 if (compound)
1239 bad += free_tail_pages_check(page, page + i);
Wei Yang534fe5e2020-06-03 15:58:36 -07001240 if (unlikely(check_free_page(page + i))) {
Mel Gormane2769db2016-05-19 17:14:38 -07001241 bad++;
1242 continue;
1243 }
1244 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1245 }
1246 }
Minchan Kimbda807d2016-07-26 15:23:05 -07001247 if (PageMappingFlags(page))
Mel Gormane2769db2016-05-19 17:14:38 -07001248 page->mapping = NULL;
Vladimir Davydovc4159a72016-08-08 23:03:12 +03001249 if (memcg_kmem_enabled() && PageKmemcg(page))
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07001250 __memcg_kmem_uncharge_page(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001251 if (check_free)
Wei Yang534fe5e2020-06-03 15:58:36 -07001252 bad += check_free_page(page);
Mel Gormane2769db2016-05-19 17:14:38 -07001253 if (bad)
1254 return false;
1255
1256 page_cpupid_reset_last(page);
1257 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1258 reset_page_owner(page, order);
1259
1260 if (!PageHighMem(page)) {
1261 debug_check_no_locks_freed(page_address(page),
1262 PAGE_SIZE << order);
1263 debug_check_no_obj_freed(page_address(page),
1264 PAGE_SIZE << order);
1265 }
Alexander Potapenko64713842019-07-11 20:59:19 -07001266 if (want_init_on_free())
1267 kernel_init_free_pages(page, 1 << order);
1268
Mel Gormane2769db2016-05-19 17:14:38 -07001269 kernel_poison_pages(page, 1 << order, 0);
Qian Cai234fdce2019-10-06 17:58:25 -07001270 /*
1271 * arch_free_page() can make the page's contents inaccessible. s390
1272 * does this. So nothing which can access the page's contents should
1273 * happen after this.
1274 */
1275 arch_free_page(page, order);
1276
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001277 if (debug_pagealloc_enabled_static())
Rick Edgecombed6332692019-04-25 17:11:35 -07001278 kernel_map_pages(page, 1 << order, 0);
1279
Waiman Long3c0c12c2018-12-28 00:38:51 -08001280 kasan_free_nondeferred_pages(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001281
1282 return true;
1283}
Mel Gorman4db75482016-05-19 17:14:32 -07001284
1285#ifdef CONFIG_DEBUG_VM
Vlastimil Babka4462b322019-07-11 20:55:09 -07001286/*
1287 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
1288 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when
1289 * moved from pcp lists to free lists.
1290 */
1291static bool free_pcp_prepare(struct page *page)
Mel Gorman4db75482016-05-19 17:14:32 -07001292{
Mel Gormane2769db2016-05-19 17:14:38 -07001293 return free_pages_prepare(page, 0, true);
Mel Gorman4db75482016-05-19 17:14:32 -07001294}
1295
Vlastimil Babka4462b322019-07-11 20:55:09 -07001296static bool bulkfree_pcp_prepare(struct page *page)
Mel Gorman4db75482016-05-19 17:14:32 -07001297{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001298 if (debug_pagealloc_enabled_static())
Wei Yang534fe5e2020-06-03 15:58:36 -07001299 return check_free_page(page);
Vlastimil Babka4462b322019-07-11 20:55:09 -07001300 else
1301 return false;
Mel Gorman4db75482016-05-19 17:14:32 -07001302}
1303#else
Vlastimil Babka4462b322019-07-11 20:55:09 -07001304/*
1305 * With DEBUG_VM disabled, order-0 pages being freed are checked only when
1306 * moving from pcp lists to free list in order to reduce overhead. With
1307 * debug_pagealloc enabled, they are checked also immediately when being freed
1308 * to the pcp lists.
1309 */
Mel Gorman4db75482016-05-19 17:14:32 -07001310static bool free_pcp_prepare(struct page *page)
1311{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001312 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07001313 return free_pages_prepare(page, 0, true);
1314 else
1315 return free_pages_prepare(page, 0, false);
Mel Gorman4db75482016-05-19 17:14:32 -07001316}
1317
1318static bool bulkfree_pcp_prepare(struct page *page)
1319{
Wei Yang534fe5e2020-06-03 15:58:36 -07001320 return check_free_page(page);
Mel Gorman4db75482016-05-19 17:14:32 -07001321}
1322#endif /* CONFIG_DEBUG_VM */
1323
Aaron Lu97334162018-04-05 16:24:14 -07001324static inline void prefetch_buddy(struct page *page)
1325{
1326 unsigned long pfn = page_to_pfn(page);
1327 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1328 struct page *buddy = page + (buddy_pfn - pfn);
1329
1330 prefetch(buddy);
1331}
1332
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333/*
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001334 * Frees a number of pages from the PCP lists
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 * Assumes all pages on list are in same zone, and of same order.
Renaud Lienhart207f36e2005-09-10 00:26:59 -07001336 * count is the number of pages to free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 *
1338 * If the zone was previously in an "all pages pinned" state then look to
1339 * see if this freeing clears that state.
1340 *
1341 * And clear the zone's pages_scanned counter, to hold off the "all pages are
1342 * pinned" detection logic.
1343 */
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001344static void free_pcppages_bulk(struct zone *zone, int count,
1345 struct per_cpu_pages *pcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346{
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001347 int migratetype = 0;
Mel Gormana6f9edd62009-09-21 17:03:20 -07001348 int batch_free = 0;
Aaron Lu97334162018-04-05 16:24:14 -07001349 int prefetch_nr = 0;
Mel Gorman37779992016-05-19 17:13:58 -07001350 bool isolated_pageblocks;
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001351 struct page *page, *tmp;
1352 LIST_HEAD(head);
Mel Gormanf2260e62009-06-16 15:32:13 -07001353
Charan Teja Reddy88e8ac12020-08-20 17:42:27 -07001354 /*
1355 * Ensure proper count is passed which otherwise would stuck in the
1356 * below while (list_empty(list)) loop.
1357 */
1358 count = min(pcp->count, count);
Mel Gormane5b31ac2016-05-19 17:14:24 -07001359 while (count) {
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001360 struct list_head *list;
Nick Piggin48db57f2006-01-08 01:00:42 -08001361
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001362 /*
Mel Gormana6f9edd62009-09-21 17:03:20 -07001363 * Remove pages from lists in a round-robin fashion. A
1364 * batch_free count is maintained that is incremented when an
1365 * empty list is encountered. This is so more pages are freed
1366 * off fuller lists instead of spinning excessively around empty
1367 * lists
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001368 */
1369 do {
Mel Gormana6f9edd62009-09-21 17:03:20 -07001370 batch_free++;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001371 if (++migratetype == MIGRATE_PCPTYPES)
1372 migratetype = 0;
1373 list = &pcp->lists[migratetype];
1374 } while (list_empty(list));
1375
Namhyung Kim1d168712011-03-22 16:32:45 -07001376 /* This is the only non-empty list. Free them all. */
1377 if (batch_free == MIGRATE_PCPTYPES)
Mel Gormane5b31ac2016-05-19 17:14:24 -07001378 batch_free = count;
Namhyung Kim1d168712011-03-22 16:32:45 -07001379
Mel Gormana6f9edd62009-09-21 17:03:20 -07001380 do {
Geliang Tanga16601c2016-01-14 15:20:30 -08001381 page = list_last_entry(list, struct page, lru);
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001382 /* must delete to avoid corrupting pcp list */
Mel Gormana6f9edd62009-09-21 17:03:20 -07001383 list_del(&page->lru);
Aaron Lu77ba9062018-04-05 16:24:06 -07001384 pcp->count--;
Vlastimil Babkaaa016d12015-09-08 15:01:22 -07001385
Mel Gorman4db75482016-05-19 17:14:32 -07001386 if (bulkfree_pcp_prepare(page))
1387 continue;
1388
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001389 list_add_tail(&page->lru, &head);
Aaron Lu97334162018-04-05 16:24:14 -07001390
1391 /*
1392 * We are going to put the page back to the global
1393 * pool, prefetch its buddy to speed up later access
1394 * under zone->lock. It is believed the overhead of
1395 * an additional test and calculating buddy_pfn here
1396 * can be offset by reduced memory latency later. To
1397 * avoid excessive prefetching due to large count, only
1398 * prefetch buddy for the first pcp->batch nr of pages.
1399 */
1400 if (prefetch_nr++ < pcp->batch)
1401 prefetch_buddy(page);
Mel Gormane5b31ac2016-05-19 17:14:24 -07001402 } while (--count && --batch_free && !list_empty(list));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403 }
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001404
1405 spin_lock(&zone->lock);
1406 isolated_pageblocks = has_isolate_pageblock(zone);
1407
1408 /*
1409 * Use safe version since after __free_one_page(),
1410 * page->lru.next will not point to original list.
1411 */
1412 list_for_each_entry_safe(page, tmp, &head, lru) {
1413 int mt = get_pcppage_migratetype(page);
1414 /* MIGRATE_ISOLATE page should not go to pcplists */
1415 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1416 /* Pageblock could have been isolated meanwhile */
1417 if (unlikely(isolated_pageblocks))
1418 mt = get_pageblock_migratetype(page);
1419
David Hildenbrandf04a5d52020-10-15 20:09:20 -07001420 __free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE);
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001421 trace_mm_page_pcpu_drain(page, 0, mt);
1422 }
Mel Gormand34b0732017-04-20 14:37:43 -07001423 spin_unlock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424}
1425
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001426static void free_one_page(struct zone *zone,
1427 struct page *page, unsigned long pfn,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07001428 unsigned int order,
David Hildenbrand7fef4312020-10-15 20:09:35 -07001429 int migratetype, fpi_t fpi_flags)
Nick Piggin48db57f2006-01-08 01:00:42 -08001430{
Mel Gormand34b0732017-04-20 14:37:43 -07001431 spin_lock(&zone->lock);
Joonsoo Kimad53f922014-11-13 15:19:11 -08001432 if (unlikely(has_isolate_pageblock(zone) ||
1433 is_migrate_isolate(migratetype))) {
1434 migratetype = get_pfnblock_migratetype(page, pfn);
Joonsoo Kimad53f922014-11-13 15:19:11 -08001435 }
David Hildenbrand7fef4312020-10-15 20:09:35 -07001436 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
Mel Gormand34b0732017-04-20 14:37:43 -07001437 spin_unlock(&zone->lock);
Nick Piggin48db57f2006-01-08 01:00:42 -08001438}
1439
Robin Holt1e8ce832015-06-30 14:56:45 -07001440static void __meminit __init_single_page(struct page *page, unsigned long pfn,
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001441 unsigned long zone, int nid)
Robin Holt1e8ce832015-06-30 14:56:45 -07001442{
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001443 mm_zero_struct_page(page);
Robin Holt1e8ce832015-06-30 14:56:45 -07001444 set_page_links(page, zone, nid, pfn);
Robin Holt1e8ce832015-06-30 14:56:45 -07001445 init_page_count(page);
1446 page_mapcount_reset(page);
1447 page_cpupid_reset_last(page);
Andrey Konovalov2813b9c2018-12-28 00:30:57 -08001448 page_kasan_tag_reset(page);
Robin Holt1e8ce832015-06-30 14:56:45 -07001449
Robin Holt1e8ce832015-06-30 14:56:45 -07001450 INIT_LIST_HEAD(&page->lru);
1451#ifdef WANT_PAGE_VIRTUAL
1452 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1453 if (!is_highmem_idx(zone))
1454 set_page_address(page, __va(pfn << PAGE_SHIFT));
1455#endif
1456}
1457
Mel Gorman7e18adb2015-06-30 14:57:05 -07001458#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Arnd Bergmann57148a62017-10-03 16:15:10 -07001459static void __meminit init_reserved_page(unsigned long pfn)
Mel Gorman7e18adb2015-06-30 14:57:05 -07001460{
1461 pg_data_t *pgdat;
1462 int nid, zid;
1463
1464 if (!early_page_uninitialised(pfn))
1465 return;
1466
1467 nid = early_pfn_to_nid(pfn);
1468 pgdat = NODE_DATA(nid);
1469
1470 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1471 struct zone *zone = &pgdat->node_zones[zid];
1472
1473 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1474 break;
1475 }
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001476 __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001477}
1478#else
1479static inline void init_reserved_page(unsigned long pfn)
1480{
1481}
1482#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1483
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001484/*
1485 * Initialised pages do not have PageReserved set. This function is
1486 * called for each range allocated by the bootmem allocator and
1487 * marks the pages PageReserved. The remaining valid pages are later
1488 * sent to the buddy page allocator.
1489 */
Stefan Bader4b50bcc2016-05-20 16:58:38 -07001490void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001491{
1492 unsigned long start_pfn = PFN_DOWN(start);
1493 unsigned long end_pfn = PFN_UP(end);
1494
Mel Gorman7e18adb2015-06-30 14:57:05 -07001495 for (; start_pfn < end_pfn; start_pfn++) {
1496 if (pfn_valid(start_pfn)) {
1497 struct page *page = pfn_to_page(start_pfn);
1498
1499 init_reserved_page(start_pfn);
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -08001500
1501 /* Avoid false-positive PageTail() */
1502 INIT_LIST_HEAD(&page->lru);
1503
Alexander Duyckd483da52018-10-26 15:07:48 -07001504 /*
1505 * no need for atomic set_bit because the struct
1506 * page is not visible yet so nobody should
1507 * access it yet.
1508 */
1509 __SetPageReserved(page);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001510 }
1511 }
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001512}
1513
David Hildenbrand7fef4312020-10-15 20:09:35 -07001514static void __free_pages_ok(struct page *page, unsigned int order,
1515 fpi_t fpi_flags)
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001516{
Mel Gormand34b0732017-04-20 14:37:43 -07001517 unsigned long flags;
Minchan Kim95e34412012-10-08 16:32:11 -07001518 int migratetype;
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001519 unsigned long pfn = page_to_pfn(page);
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001520
Mel Gormane2769db2016-05-19 17:14:38 -07001521 if (!free_pages_prepare(page, order, true))
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001522 return;
1523
Mel Gormancfc47a22014-06-04 16:10:19 -07001524 migratetype = get_pfnblock_migratetype(page, pfn);
Mel Gormand34b0732017-04-20 14:37:43 -07001525 local_irq_save(flags);
1526 __count_vm_events(PGFREE, 1 << order);
David Hildenbrand7fef4312020-10-15 20:09:35 -07001527 free_one_page(page_zone(page), page, pfn, order, migratetype,
1528 fpi_flags);
Mel Gormand34b0732017-04-20 14:37:43 -07001529 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530}
1531
Arun KSa9cd4102019-03-05 15:42:14 -08001532void __free_pages_core(struct page *page, unsigned int order)
David Howellsa226f6c2006-01-06 00:11:08 -08001533{
Johannes Weinerc3993072012-01-10 15:08:10 -08001534 unsigned int nr_pages = 1 << order;
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001535 struct page *p = page;
Johannes Weinerc3993072012-01-10 15:08:10 -08001536 unsigned int loop;
David Howellsa226f6c2006-01-06 00:11:08 -08001537
David Hildenbrand7fef4312020-10-15 20:09:35 -07001538 /*
1539 * When initializing the memmap, __init_single_page() sets the refcount
1540 * of all pages to 1 ("allocated"/"not free"). We have to set the
1541 * refcount of all involved pages to 0.
1542 */
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001543 prefetchw(p);
1544 for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1545 prefetchw(p + 1);
Johannes Weinerc3993072012-01-10 15:08:10 -08001546 __ClearPageReserved(p);
1547 set_page_count(p, 0);
David Howellsa226f6c2006-01-06 00:11:08 -08001548 }
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001549 __ClearPageReserved(p);
1550 set_page_count(p, 0);
Johannes Weinerc3993072012-01-10 15:08:10 -08001551
Arun KS9705bea2018-12-28 00:34:24 -08001552 atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
David Hildenbrand7fef4312020-10-15 20:09:35 -07001553
1554 /*
1555 * Bypass PCP and place fresh pages right to the tail, primarily
1556 * relevant for memory onlining.
1557 */
1558 __free_pages_ok(page, order, FPI_TO_TAIL);
David Howellsa226f6c2006-01-06 00:11:08 -08001559}
1560
Mike Rapoport3f08a302020-06-03 15:57:02 -07001561#ifdef CONFIG_NEED_MULTIPLE_NODES
Mel Gorman7ace9912015-08-06 15:46:13 -07001562
Mel Gorman75a592a2015-06-30 14:56:59 -07001563static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1564
Mike Rapoport6f24fbd2020-06-03 15:56:57 -07001565#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1566
1567/*
1568 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
1569 */
1570int __meminit __early_pfn_to_nid(unsigned long pfn,
1571 struct mminit_pfnnid_cache *state)
1572{
1573 unsigned long start_pfn, end_pfn;
1574 int nid;
1575
1576 if (state->last_start <= pfn && pfn < state->last_end)
1577 return state->last_nid;
1578
1579 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
1580 if (nid != NUMA_NO_NODE) {
1581 state->last_start = start_pfn;
1582 state->last_end = end_pfn;
1583 state->last_nid = nid;
1584 }
1585
1586 return nid;
1587}
1588#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
1589
Mel Gorman75a592a2015-06-30 14:56:59 -07001590int __meminit early_pfn_to_nid(unsigned long pfn)
1591{
Mel Gorman7ace9912015-08-06 15:46:13 -07001592 static DEFINE_SPINLOCK(early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001593 int nid;
1594
Mel Gorman7ace9912015-08-06 15:46:13 -07001595 spin_lock(&early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001596 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
Mel Gorman7ace9912015-08-06 15:46:13 -07001597 if (nid < 0)
Mel Gormane4568d32016-07-14 12:07:20 -07001598 nid = first_online_node;
Mel Gorman7ace9912015-08-06 15:46:13 -07001599 spin_unlock(&early_pfn_lock);
1600
1601 return nid;
Mel Gorman75a592a2015-06-30 14:56:59 -07001602}
Mike Rapoport3f08a302020-06-03 15:57:02 -07001603#endif /* CONFIG_NEED_MULTIPLE_NODES */
Mel Gorman75a592a2015-06-30 14:56:59 -07001604
Mike Rapoport7c2ee342018-10-30 15:09:36 -07001605void __init memblock_free_pages(struct page *page, unsigned long pfn,
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001606 unsigned int order)
1607{
1608 if (early_page_uninitialised(pfn))
1609 return;
Arun KSa9cd4102019-03-05 15:42:14 -08001610 __free_pages_core(page, order);
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001611}
1612
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001613/*
1614 * Check that the whole (or subset of) a pageblock given by the interval of
1615 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1616 * with the migration of free compaction scanner. The scanners then need to
1617 * use only pfn_valid_within() check for arches that allow holes within
1618 * pageblocks.
1619 *
1620 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1621 *
1622 * It's possible on some configurations to have a setup like node0 node1 node0
1623 * i.e. it's possible that all pages within a zones range of pages do not
1624 * belong to a single zone. We assume that a border between node0 and node1
1625 * can occur within a single pageblock, but not a node0 node1 node0
1626 * interleaving within a single pageblock. It is therefore sufficient to check
1627 * the first and last page of a pageblock and avoid checking each individual
1628 * page in a pageblock.
1629 */
1630struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1631 unsigned long end_pfn, struct zone *zone)
1632{
1633 struct page *start_page;
1634 struct page *end_page;
1635
1636 /* end_pfn is one past the range we are checking */
1637 end_pfn--;
1638
1639 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1640 return NULL;
1641
Michal Hocko2d070ea2017-07-06 15:37:56 -07001642 start_page = pfn_to_online_page(start_pfn);
1643 if (!start_page)
1644 return NULL;
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001645
1646 if (page_zone(start_page) != zone)
1647 return NULL;
1648
1649 end_page = pfn_to_page(end_pfn);
1650
1651 /* This gives a shorter code than deriving page_zone(end_page) */
1652 if (page_zone_id(start_page) != page_zone_id(end_page))
1653 return NULL;
1654
1655 return start_page;
1656}
1657
1658void set_zone_contiguous(struct zone *zone)
1659{
1660 unsigned long block_start_pfn = zone->zone_start_pfn;
1661 unsigned long block_end_pfn;
1662
1663 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1664 for (; block_start_pfn < zone_end_pfn(zone);
1665 block_start_pfn = block_end_pfn,
1666 block_end_pfn += pageblock_nr_pages) {
1667
1668 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1669
1670 if (!__pageblock_pfn_to_page(block_start_pfn,
1671 block_end_pfn, zone))
1672 return;
David Hildenbrande84fe992020-05-07 18:35:46 -07001673 cond_resched();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001674 }
1675
1676 /* We confirm that there is no hole */
1677 zone->contiguous = true;
1678}
1679
1680void clear_zone_contiguous(struct zone *zone)
1681{
1682 zone->contiguous = false;
1683}
1684
Mel Gorman7e18adb2015-06-30 14:57:05 -07001685#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001686static void __init deferred_free_range(unsigned long pfn,
1687 unsigned long nr_pages)
Mel Gormana4de83d2015-06-30 14:57:16 -07001688{
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001689 struct page *page;
1690 unsigned long i;
Mel Gormana4de83d2015-06-30 14:57:16 -07001691
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001692 if (!nr_pages)
Mel Gormana4de83d2015-06-30 14:57:16 -07001693 return;
1694
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001695 page = pfn_to_page(pfn);
1696
Mel Gormana4de83d2015-06-30 14:57:16 -07001697 /* Free a large naturally-aligned chunk if possible */
Xishi Qiue7801492016-10-07 16:58:09 -07001698 if (nr_pages == pageblock_nr_pages &&
1699 (pfn & (pageblock_nr_pages - 1)) == 0) {
Mel Gormanac5d2532015-06-30 14:57:20 -07001700 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Arun KSa9cd4102019-03-05 15:42:14 -08001701 __free_pages_core(page, pageblock_order);
Mel Gormana4de83d2015-06-30 14:57:16 -07001702 return;
1703 }
1704
Xishi Qiue7801492016-10-07 16:58:09 -07001705 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1706 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1707 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Arun KSa9cd4102019-03-05 15:42:14 -08001708 __free_pages_core(page, 0);
Xishi Qiue7801492016-10-07 16:58:09 -07001709 }
Mel Gormana4de83d2015-06-30 14:57:16 -07001710}
1711
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001712/* Completion tracking for deferred_init_memmap() threads */
1713static atomic_t pgdat_init_n_undone __initdata;
1714static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1715
1716static inline void __init pgdat_init_report_one_done(void)
1717{
1718 if (atomic_dec_and_test(&pgdat_init_n_undone))
1719 complete(&pgdat_init_all_done_comp);
1720}
Mel Gorman0e1cc952015-06-30 14:57:27 -07001721
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001722/*
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001723 * Returns true if page needs to be initialized or freed to buddy allocator.
1724 *
1725 * First we check if pfn is valid on architectures where it is possible to have
1726 * holes within pageblock_nr_pages. On systems where it is not possible, this
1727 * function is optimized out.
1728 *
1729 * Then, we check if a current large page is valid by only checking the validity
1730 * of the head pfn.
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001731 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001732static inline bool __init deferred_pfn_valid(unsigned long pfn)
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001733{
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001734 if (!pfn_valid_within(pfn))
1735 return false;
1736 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1737 return false;
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001738 return true;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001739}
1740
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001741/*
1742 * Free pages to buddy allocator. Try to free aligned pages in
1743 * pageblock_nr_pages sizes.
1744 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001745static void __init deferred_free_pages(unsigned long pfn,
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001746 unsigned long end_pfn)
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001747{
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001748 unsigned long nr_pgmask = pageblock_nr_pages - 1;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001749 unsigned long nr_free = 0;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001750
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001751 for (; pfn < end_pfn; pfn++) {
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001752 if (!deferred_pfn_valid(pfn)) {
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001753 deferred_free_range(pfn - nr_free, nr_free);
1754 nr_free = 0;
1755 } else if (!(pfn & nr_pgmask)) {
1756 deferred_free_range(pfn - nr_free, nr_free);
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001757 nr_free = 1;
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001758 } else {
1759 nr_free++;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001760 }
1761 }
1762 /* Free the last block of pages to allocator */
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001763 deferred_free_range(pfn - nr_free, nr_free);
1764}
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001765
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001766/*
1767 * Initialize struct pages. We minimize pfn page lookups and scheduler checks
1768 * by performing it only once every pageblock_nr_pages.
1769 * Return number of pages initialized.
1770 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001771static unsigned long __init deferred_init_pages(struct zone *zone,
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001772 unsigned long pfn,
1773 unsigned long end_pfn)
1774{
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001775 unsigned long nr_pgmask = pageblock_nr_pages - 1;
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001776 int nid = zone_to_nid(zone);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001777 unsigned long nr_pages = 0;
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001778 int zid = zone_idx(zone);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001779 struct page *page = NULL;
1780
1781 for (; pfn < end_pfn; pfn++) {
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001782 if (!deferred_pfn_valid(pfn)) {
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001783 page = NULL;
1784 continue;
1785 } else if (!page || !(pfn & nr_pgmask)) {
1786 page = pfn_to_page(pfn);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001787 } else {
1788 page++;
1789 }
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001790 __init_single_page(page, pfn, zid, nid);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001791 nr_pages++;
1792 }
1793 return (nr_pages);
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001794}
1795
Alexander Duyck0e56aca2019-05-13 17:21:20 -07001796/*
1797 * This function is meant to pre-load the iterator for the zone init.
1798 * Specifically it walks through the ranges until we are caught up to the
1799 * first_init_pfn value and exits there. If we never encounter the value we
1800 * return false indicating there are no valid ranges left.
1801 */
1802static bool __init
1803deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
1804 unsigned long *spfn, unsigned long *epfn,
1805 unsigned long first_init_pfn)
1806{
1807 u64 j;
1808
1809 /*
1810 * Start out by walking through the ranges in this zone that have
1811 * already been initialized. We don't need to do anything with them
1812 * so we just need to flush them out of the system.
1813 */
1814 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
1815 if (*epfn <= first_init_pfn)
1816 continue;
1817 if (*spfn < first_init_pfn)
1818 *spfn = first_init_pfn;
1819 *i = j;
1820 return true;
1821 }
1822
1823 return false;
1824}
1825
1826/*
1827 * Initialize and free pages. We do it in two loops: first we initialize
1828 * struct page, then free to buddy allocator, because while we are
1829 * freeing pages we can access pages that are ahead (computing buddy
1830 * page in __free_one_page()).
1831 *
1832 * In order to try and keep some memory in the cache we have the loop
1833 * broken along max page order boundaries. This way we will not cause
1834 * any issues with the buddy page computation.
1835 */
1836static unsigned long __init
1837deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
1838 unsigned long *end_pfn)
1839{
1840 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
1841 unsigned long spfn = *start_pfn, epfn = *end_pfn;
1842 unsigned long nr_pages = 0;
1843 u64 j = *i;
1844
1845 /* First we loop through and initialize the page values */
1846 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
1847 unsigned long t;
1848
1849 if (mo_pfn <= *start_pfn)
1850 break;
1851
1852 t = min(mo_pfn, *end_pfn);
1853 nr_pages += deferred_init_pages(zone, *start_pfn, t);
1854
1855 if (mo_pfn < *end_pfn) {
1856 *start_pfn = mo_pfn;
1857 break;
1858 }
1859 }
1860
1861 /* Reset values and now loop through freeing pages as needed */
1862 swap(j, *i);
1863
1864 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
1865 unsigned long t;
1866
1867 if (mo_pfn <= spfn)
1868 break;
1869
1870 t = min(mo_pfn, epfn);
1871 deferred_free_pages(spfn, t);
1872
1873 if (mo_pfn <= epfn)
1874 break;
1875 }
1876
1877 return nr_pages;
1878}
1879
Daniel Jordane4443142020-06-03 15:59:51 -07001880static void __init
1881deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
1882 void *arg)
1883{
1884 unsigned long spfn, epfn;
1885 struct zone *zone = arg;
1886 u64 i;
1887
1888 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
1889
1890 /*
1891 * Initialize and free pages in MAX_ORDER sized increments so that we
1892 * can avoid introducing any issues with the buddy allocator.
1893 */
1894 while (spfn < end_pfn) {
1895 deferred_init_maxorder(&i, zone, &spfn, &epfn);
1896 cond_resched();
1897 }
1898}
1899
Daniel Jordanecd09652020-06-03 15:59:55 -07001900/* An arch may override for more concurrency. */
1901__weak int __init
1902deferred_page_init_max_threads(const struct cpumask *node_cpumask)
1903{
1904 return 1;
1905}
1906
Mel Gorman7e18adb2015-06-30 14:57:05 -07001907/* Initialise remaining memory on a node */
Mel Gorman0e1cc952015-06-30 14:57:27 -07001908static int __init deferred_init_memmap(void *data)
Mel Gorman7e18adb2015-06-30 14:57:05 -07001909{
Mel Gorman0e1cc952015-06-30 14:57:27 -07001910 pg_data_t *pgdat = data;
Mel Gorman0e1cc952015-06-30 14:57:27 -07001911 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
Daniel Jordan89c7c402020-06-03 15:59:47 -07001912 unsigned long spfn = 0, epfn = 0;
Alexander Duyck0e56aca2019-05-13 17:21:20 -07001913 unsigned long first_init_pfn, flags;
1914 unsigned long start = jiffies;
1915 struct zone *zone;
Daniel Jordane4443142020-06-03 15:59:51 -07001916 int zid, max_threads;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001917 u64 i;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001918
Mel Gorman0e1cc952015-06-30 14:57:27 -07001919 /* Bind memory initialisation thread to a local node if possible */
1920 if (!cpumask_empty(cpumask))
1921 set_cpus_allowed_ptr(current, cpumask);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001922
Pavel Tatashin3a2d7fa2018-04-05 16:22:27 -07001923 pgdat_resize_lock(pgdat, &flags);
1924 first_init_pfn = pgdat->first_deferred_pfn;
1925 if (first_init_pfn == ULONG_MAX) {
1926 pgdat_resize_unlock(pgdat, &flags);
1927 pgdat_init_report_one_done();
1928 return 0;
1929 }
1930
Mel Gorman7e18adb2015-06-30 14:57:05 -07001931 /* Sanity check boundaries */
1932 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1933 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1934 pgdat->first_deferred_pfn = ULONG_MAX;
1935
Pavel Tatashin3d060852020-06-03 15:59:24 -07001936 /*
1937 * Once we unlock here, the zone cannot be grown anymore, thus if an
1938 * interrupt thread must allocate this early in boot, zone must be
1939 * pre-grown prior to start of deferred page initialization.
1940 */
1941 pgdat_resize_unlock(pgdat, &flags);
1942
Mel Gorman7e18adb2015-06-30 14:57:05 -07001943 /* Only the highest zone is deferred so find it */
1944 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1945 zone = pgdat->node_zones + zid;
1946 if (first_init_pfn < zone_end_pfn(zone))
1947 break;
1948 }
Alexander Duyck0e56aca2019-05-13 17:21:20 -07001949
1950 /* If the zone is empty somebody else may have cleared out the zone */
1951 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1952 first_init_pfn))
1953 goto zone_empty;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001954
Daniel Jordanecd09652020-06-03 15:59:55 -07001955 max_threads = deferred_page_init_max_threads(cpumask);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001956
Daniel Jordan117003c2020-06-03 15:59:20 -07001957 while (spfn < epfn) {
Daniel Jordane4443142020-06-03 15:59:51 -07001958 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
1959 struct padata_mt_job job = {
1960 .thread_fn = deferred_init_memmap_chunk,
1961 .fn_arg = zone,
1962 .start = spfn,
1963 .size = epfn_align - spfn,
1964 .align = PAGES_PER_SECTION,
1965 .min_chunk = PAGES_PER_SECTION,
1966 .max_threads = max_threads,
1967 };
1968
1969 padata_do_multithreaded(&job);
1970 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1971 epfn_align);
Daniel Jordan117003c2020-06-03 15:59:20 -07001972 }
Mel Gorman7e18adb2015-06-30 14:57:05 -07001973zone_empty:
Mel Gorman7e18adb2015-06-30 14:57:05 -07001974 /* Sanity check that the next zone really is unpopulated */
1975 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1976
Daniel Jordan89c7c402020-06-03 15:59:47 -07001977 pr_info("node %d deferred pages initialised in %ums\n",
1978 pgdat->node_id, jiffies_to_msecs(jiffies - start));
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001979
1980 pgdat_init_report_one_done();
Mel Gorman0e1cc952015-06-30 14:57:27 -07001981 return 0;
1982}
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07001983
1984/*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07001985 * If this zone has deferred pages, try to grow it by initializing enough
1986 * deferred pages to satisfy the allocation specified by order, rounded up to
1987 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
1988 * of SECTION_SIZE bytes by initializing struct pages in increments of
1989 * PAGES_PER_SECTION * sizeof(struct page) bytes.
1990 *
1991 * Return true when zone was grown, otherwise return false. We return true even
1992 * when we grow less than requested, to let the caller decide if there are
1993 * enough pages to satisfy the allocation.
1994 *
1995 * Note: We use noinline because this function is needed only during boot, and
1996 * it is called from a __ref function _deferred_grow_zone. This way we are
1997 * making sure that it is not inlined into permanent text section.
1998 */
1999static noinline bool __init
2000deferred_grow_zone(struct zone *zone, unsigned int order)
2001{
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002002 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
Alexander Duyck837566e2019-05-13 17:21:17 -07002003 pg_data_t *pgdat = zone->zone_pgdat;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002004 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002005 unsigned long spfn, epfn, flags;
2006 unsigned long nr_pages = 0;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002007 u64 i;
2008
2009 /* Only the last zone may have deferred pages */
2010 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2011 return false;
2012
2013 pgdat_resize_lock(pgdat, &flags);
2014
2015 /*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002016 * If someone grew this zone while we were waiting for spinlock, return
2017 * true, as there might be enough pages already.
2018 */
2019 if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2020 pgdat_resize_unlock(pgdat, &flags);
2021 return true;
2022 }
2023
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002024 /* If the zone is empty somebody else may have cleared out the zone */
2025 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2026 first_deferred_pfn)) {
2027 pgdat->first_deferred_pfn = ULONG_MAX;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002028 pgdat_resize_unlock(pgdat, &flags);
Juergen Grossb9705d82019-07-04 15:14:36 -07002029 /* Retry only once. */
2030 return first_deferred_pfn != ULONG_MAX;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002031 }
2032
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002033 /*
2034 * Initialize and free pages in MAX_ORDER sized increments so
2035 * that we can avoid introducing any issues with the buddy
2036 * allocator.
2037 */
2038 while (spfn < epfn) {
2039 /* update our first deferred PFN for this section */
2040 first_deferred_pfn = spfn;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002041
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002042 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
Daniel Jordan117003c2020-06-03 15:59:20 -07002043 touch_nmi_watchdog();
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002044
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002045 /* We should only stop along section boundaries */
2046 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2047 continue;
2048
2049 /* If our quota has been met we can stop here */
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002050 if (nr_pages >= nr_pages_needed)
2051 break;
2052 }
2053
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002054 pgdat->first_deferred_pfn = spfn;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002055 pgdat_resize_unlock(pgdat, &flags);
2056
2057 return nr_pages > 0;
2058}
2059
2060/*
2061 * deferred_grow_zone() is __init, but it is called from
2062 * get_page_from_freelist() during early boot until deferred_pages permanently
2063 * disables this call. This is why we have refdata wrapper to avoid warning,
2064 * and to ensure that the function body gets unloaded.
2065 */
2066static bool __ref
2067_deferred_grow_zone(struct zone *zone, unsigned int order)
2068{
2069 return deferred_grow_zone(zone, order);
2070}
2071
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002072#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
Mel Gorman0e1cc952015-06-30 14:57:27 -07002073
2074void __init page_alloc_init_late(void)
2075{
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002076 struct zone *zone;
Dan Williamse900a912019-05-14 15:41:28 -07002077 int nid;
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002078
2079#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Mel Gorman0e1cc952015-06-30 14:57:27 -07002080
Nicolai Stanged3cd1312015-08-06 15:46:16 -07002081 /* There will be num_node_state(N_MEMORY) threads */
2082 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
Mel Gorman0e1cc952015-06-30 14:57:27 -07002083 for_each_node_state(nid, N_MEMORY) {
Mel Gorman0e1cc952015-06-30 14:57:27 -07002084 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2085 }
2086
2087 /* Block until all are initialised */
Nicolai Stanged3cd1312015-08-06 15:46:16 -07002088 wait_for_completion(&pgdat_init_all_done_comp);
Mel Gorman4248b0d2015-08-06 15:46:20 -07002089
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002090 /*
Mel Gorman3e8fc002019-11-05 21:16:27 -08002091 * The number of managed pages has changed due to the initialisation
2092 * so the pcpu batch and high limits needs to be updated or the limits
2093 * will be artificially small.
2094 */
2095 for_each_populated_zone(zone)
2096 zone_pcp_update(zone);
2097
2098 /*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002099 * We initialized the rest of the deferred pages. Permanently disable
2100 * on-demand struct page initialization.
2101 */
2102 static_branch_disable(&deferred_pages);
2103
Mel Gorman4248b0d2015-08-06 15:46:20 -07002104 /* Reinit limits that are based on free pages after the kernel is up */
2105 files_maxfiles_init();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002106#endif
Mike Rapoport350e88b2019-05-13 17:22:59 -07002107
Pavel Tatashin3010f872017-08-18 15:16:05 -07002108 /* Discard memblock private memory */
2109 memblock_discard();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002110
Dan Williamse900a912019-05-14 15:41:28 -07002111 for_each_node_state(nid, N_MEMORY)
2112 shuffle_free_memory(NODE_DATA(nid));
2113
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002114 for_each_populated_zone(zone)
2115 set_zone_contiguous(zone);
Mel Gorman7e18adb2015-06-30 14:57:05 -07002116}
Mel Gorman7e18adb2015-06-30 14:57:05 -07002117
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002118#ifdef CONFIG_CMA
Li Zhong9cf510a2013-08-23 13:52:52 +08002119/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002120void __init init_cma_reserved_pageblock(struct page *page)
2121{
2122 unsigned i = pageblock_nr_pages;
2123 struct page *p = page;
2124
2125 do {
2126 __ClearPageReserved(p);
2127 set_page_count(p, 0);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09002128 } while (++p, --i);
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002129
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002130 set_pageblock_migratetype(page, MIGRATE_CMA);
Michal Nazarewiczdc783272014-07-02 15:22:35 -07002131
2132 if (pageblock_order >= MAX_ORDER) {
2133 i = pageblock_nr_pages;
2134 p = page;
2135 do {
2136 set_page_refcounted(p);
2137 __free_pages(p, MAX_ORDER - 1);
2138 p += MAX_ORDER_NR_PAGES;
2139 } while (i -= MAX_ORDER_NR_PAGES);
2140 } else {
2141 set_page_refcounted(page);
2142 __free_pages(page, pageblock_order);
2143 }
2144
Jiang Liu3dcc0572013-07-03 15:03:21 -07002145 adjust_managed_page_count(page, pageblock_nr_pages);
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002146}
2147#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148
2149/*
2150 * The order of subdivision here is critical for the IO subsystem.
2151 * Please do not alter this order without good reasons and regression
2152 * testing. Specifically, as large blocks of memory are subdivided,
2153 * the order in which smaller blocks are delivered depends on the order
2154 * they're subdivided in this function. This is the primary factor
2155 * influencing the order in which pages are delivered to the IO
2156 * subsystem according to empirical testing, and this is also justified
2157 * by considering the behavior of a buddy system containing a single
2158 * large block of memory acted on by a series of small allocations.
2159 * This behavior is a critical factor in sglist merging's success.
2160 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +01002161 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -07002162 */
Nick Piggin085cc7d52006-01-06 00:11:01 -08002163static inline void expand(struct zone *zone, struct page *page,
Alexander Duyck6ab01362020-04-06 20:04:49 -07002164 int low, int high, int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002165{
2166 unsigned long size = 1 << high;
2167
2168 while (high > low) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002169 high--;
2170 size >>= 1;
Sasha Levin309381fea2014-01-23 15:52:54 -08002171 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08002172
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07002173 /*
2174 * Mark as guard pages (or page), that will allow to
2175 * merge back to allocator when buddy will be freed.
2176 * Corresponding page table entries will not be touched,
2177 * pages will stay not present in virtual address space
2178 */
2179 if (set_page_guard(zone, &page[size], high, migratetype))
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08002180 continue;
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07002181
Alexander Duyck6ab01362020-04-06 20:04:49 -07002182 add_to_free_list(&page[size], zone, high, migratetype);
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002183 set_buddy_order(&page[size], high);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002184 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002185}
2186
Vlastimil Babka4e611802016-05-19 17:14:41 -07002187static void check_new_page_bad(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002188{
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07002189 if (unlikely(page->flags & __PG_HWPOISON)) {
Naoya Horiguchie570f562016-05-20 16:58:50 -07002190 /* Don't complain about hwpoisoned pages */
2191 page_mapcount_reset(page); /* remove PageBuddy */
2192 return;
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07002193 }
Wei Yang58b7f112020-06-03 15:58:39 -07002194
2195 bad_page(page,
2196 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
Vlastimil Babka4e611802016-05-19 17:14:41 -07002197}
2198
2199/*
2200 * This page is about to be returned from the page allocator
2201 */
2202static inline int check_new_page(struct page *page)
2203{
2204 if (likely(page_expected_state(page,
2205 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
2206 return 0;
2207
2208 check_new_page_bad(page);
2209 return 1;
Wu Fengguang2a7684a2009-09-16 11:50:12 +02002210}
2211
Vinayak Menonbd33ef32017-05-03 14:54:42 -07002212static inline bool free_pages_prezeroed(void)
Laura Abbott1414c7f2016-03-15 14:56:30 -07002213{
Alexander Potapenko64713842019-07-11 20:59:19 -07002214 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
2215 page_poisoning_enabled()) || want_init_on_free();
Laura Abbott1414c7f2016-03-15 14:56:30 -07002216}
2217
Mel Gorman479f8542016-05-19 17:14:35 -07002218#ifdef CONFIG_DEBUG_VM
Vlastimil Babka4462b322019-07-11 20:55:09 -07002219/*
2220 * With DEBUG_VM enabled, order-0 pages are checked for expected state when
2221 * being allocated from pcp lists. With debug_pagealloc also enabled, they are
2222 * also checked when pcp lists are refilled from the free lists.
2223 */
2224static inline bool check_pcp_refill(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002225{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002226 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07002227 return check_new_page(page);
2228 else
2229 return false;
Mel Gorman479f8542016-05-19 17:14:35 -07002230}
2231
Vlastimil Babka4462b322019-07-11 20:55:09 -07002232static inline bool check_new_pcp(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002233{
2234 return check_new_page(page);
2235}
2236#else
Vlastimil Babka4462b322019-07-11 20:55:09 -07002237/*
2238 * With DEBUG_VM disabled, free order-0 pages are checked for expected state
2239 * when pcp lists are being refilled from the free lists. With debug_pagealloc
2240 * enabled, they are also checked when being allocated from the pcp lists.
2241 */
2242static inline bool check_pcp_refill(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002243{
2244 return check_new_page(page);
2245}
Vlastimil Babka4462b322019-07-11 20:55:09 -07002246static inline bool check_new_pcp(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002247{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002248 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07002249 return check_new_page(page);
2250 else
2251 return false;
Mel Gorman479f8542016-05-19 17:14:35 -07002252}
2253#endif /* CONFIG_DEBUG_VM */
2254
2255static bool check_new_pages(struct page *page, unsigned int order)
2256{
2257 int i;
2258 for (i = 0; i < (1 << order); i++) {
2259 struct page *p = page + i;
2260
2261 if (unlikely(check_new_page(p)))
2262 return true;
2263 }
2264
2265 return false;
2266}
2267
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002268inline void post_alloc_hook(struct page *page, unsigned int order,
2269 gfp_t gfp_flags)
2270{
2271 set_page_private(page, 0);
2272 set_page_refcounted(page);
2273
2274 arch_alloc_page(page, order);
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002275 if (debug_pagealloc_enabled_static())
Rick Edgecombed6332692019-04-25 17:11:35 -07002276 kernel_map_pages(page, 1 << order, 1);
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002277 kasan_alloc_pages(page, order);
Qian Cai41179922019-03-05 15:41:24 -08002278 kernel_poison_pages(page, 1 << order, 1);
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002279 set_page_owner(page, order, gfp_flags);
2280}
2281
Mel Gorman479f8542016-05-19 17:14:35 -07002282static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Mel Gormanc6038442016-05-19 17:13:38 -07002283 unsigned int alloc_flags)
Wu Fengguang2a7684a2009-09-16 11:50:12 +02002284{
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002285 post_alloc_hook(page, order, gfp_flags);
Nick Piggin17cf4402006-03-22 00:08:41 -08002286
Alexander Potapenko64713842019-07-11 20:59:19 -07002287 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags))
2288 kernel_init_free_pages(page, 1 << order);
Nick Piggin17cf4402006-03-22 00:08:41 -08002289
2290 if (order && (gfp_flags & __GFP_COMP))
2291 prep_compound_page(page, order);
2292
Vlastimil Babka75379192015-02-11 15:25:38 -08002293 /*
Michal Hocko2f064f32015-08-21 14:11:51 -07002294 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
Vlastimil Babka75379192015-02-11 15:25:38 -08002295 * allocate the page. The expectation is that the caller is taking
2296 * steps that will free more memory. The caller should avoid the page
2297 * being used for !PFMEMALLOC purposes.
2298 */
Michal Hocko2f064f32015-08-21 14:11:51 -07002299 if (alloc_flags & ALLOC_NO_WATERMARKS)
2300 set_page_pfmemalloc(page);
2301 else
2302 clear_page_pfmemalloc(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303}
2304
Mel Gorman56fd56b2007-10-16 01:25:58 -07002305/*
2306 * Go through the free lists for the given migratetype and remove
2307 * the smallest available page from the freelists
2308 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002309static __always_inline
Mel Gorman728ec982009-06-16 15:32:04 -07002310struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
Mel Gorman56fd56b2007-10-16 01:25:58 -07002311 int migratetype)
2312{
2313 unsigned int current_order;
Pintu Kumarb8af2942013-09-11 14:20:34 -07002314 struct free_area *area;
Mel Gorman56fd56b2007-10-16 01:25:58 -07002315 struct page *page;
2316
2317 /* Find a page of the appropriate size in the preferred list */
2318 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2319 area = &(zone->free_area[current_order]);
Dan Williamsb03641a2019-05-14 15:41:32 -07002320 page = get_page_from_free_area(area, migratetype);
Geliang Tanga16601c2016-01-14 15:20:30 -08002321 if (!page)
2322 continue;
Alexander Duyck6ab01362020-04-06 20:04:49 -07002323 del_page_from_free_list(page, zone, current_order);
2324 expand(zone, page, order, current_order, migratetype);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002325 set_pcppage_migratetype(page, migratetype);
Mel Gorman56fd56b2007-10-16 01:25:58 -07002326 return page;
2327 }
2328
2329 return NULL;
2330}
2331
2332
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002333/*
2334 * This array describes the order lists are fallen back to when
2335 * the free lists for the desirable migrate type are depleted
2336 */
Wei Yangda415662020-08-06 23:25:58 -07002337static int fallbacks[MIGRATE_TYPES][3] = {
Mel Gorman974a7862015-11-06 16:28:34 -08002338 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
Mel Gorman974a7862015-11-06 16:28:34 -08002339 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
Huang Shijie7ead3342018-12-28 00:34:46 -08002340 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
Joonsoo Kimdc676472015-04-14 15:45:15 -07002341#ifdef CONFIG_CMA
Mel Gorman974a7862015-11-06 16:28:34 -08002342 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002343#endif
Minchan Kim194159f2013-02-22 16:33:58 -08002344#ifdef CONFIG_MEMORY_ISOLATION
Mel Gorman974a7862015-11-06 16:28:34 -08002345 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */
Minchan Kim194159f2013-02-22 16:33:58 -08002346#endif
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002347};
2348
Joonsoo Kimdc676472015-04-14 15:45:15 -07002349#ifdef CONFIG_CMA
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002350static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
Joonsoo Kimdc676472015-04-14 15:45:15 -07002351 unsigned int order)
2352{
2353 return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2354}
2355#else
2356static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2357 unsigned int order) { return NULL; }
2358#endif
2359
Mel Gormanc361be52007-10-16 01:25:51 -07002360/*
David Hildenbrand293ffa52020-10-15 20:09:30 -07002361 * Move the free pages in a range to the freelist tail of the requested type.
Mel Gormand9c23402007-10-16 01:26:01 -07002362 * Note that start_page and end_pages are not aligned on a pageblock
Mel Gormanc361be52007-10-16 01:25:51 -07002363 * boundary. If alignment is required, use move_freepages_block()
2364 */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002365static int move_freepages(struct zone *zone,
Adrian Bunkb69a7282008-07-23 21:28:12 -07002366 struct page *start_page, struct page *end_page,
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002367 int migratetype, int *num_movable)
Mel Gormanc361be52007-10-16 01:25:51 -07002368{
2369 struct page *page;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08002370 unsigned int order;
Mel Gormand1003132007-10-16 01:26:00 -07002371 int pages_moved = 0;
Mel Gormanc361be52007-10-16 01:25:51 -07002372
Mel Gormanc361be52007-10-16 01:25:51 -07002373 for (page = start_page; page <= end_page;) {
2374 if (!pfn_valid_within(page_to_pfn(page))) {
2375 page++;
2376 continue;
2377 }
2378
2379 if (!PageBuddy(page)) {
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002380 /*
2381 * We assume that pages that could be isolated for
2382 * migration are movable. But we don't actually try
2383 * isolating, as that would be expensive.
2384 */
2385 if (num_movable &&
2386 (PageLRU(page) || __PageMovable(page)))
2387 (*num_movable)++;
2388
Mel Gormanc361be52007-10-16 01:25:51 -07002389 page++;
2390 continue;
2391 }
2392
David Rientjescd961032019-08-24 17:54:40 -07002393 /* Make sure we are not inadvertently changing nodes */
2394 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2395 VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2396
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002397 order = buddy_order(page);
Alexander Duyck6ab01362020-04-06 20:04:49 -07002398 move_to_free_list(page, zone, order, migratetype);
Mel Gormanc361be52007-10-16 01:25:51 -07002399 page += 1 << order;
Mel Gormand1003132007-10-16 01:26:00 -07002400 pages_moved += 1 << order;
Mel Gormanc361be52007-10-16 01:25:51 -07002401 }
2402
Mel Gormand1003132007-10-16 01:26:00 -07002403 return pages_moved;
Mel Gormanc361be52007-10-16 01:25:51 -07002404}
2405
Minchan Kimee6f5092012-07-31 16:43:50 -07002406int move_freepages_block(struct zone *zone, struct page *page,
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002407 int migratetype, int *num_movable)
Mel Gormanc361be52007-10-16 01:25:51 -07002408{
2409 unsigned long start_pfn, end_pfn;
2410 struct page *start_page, *end_page;
2411
David Rientjes4a222122018-10-26 15:09:24 -07002412 if (num_movable)
2413 *num_movable = 0;
2414
Mel Gormanc361be52007-10-16 01:25:51 -07002415 start_pfn = page_to_pfn(page);
Mel Gormand9c23402007-10-16 01:26:01 -07002416 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
Mel Gormanc361be52007-10-16 01:25:51 -07002417 start_page = pfn_to_page(start_pfn);
Mel Gormand9c23402007-10-16 01:26:01 -07002418 end_page = start_page + pageblock_nr_pages - 1;
2419 end_pfn = start_pfn + pageblock_nr_pages - 1;
Mel Gormanc361be52007-10-16 01:25:51 -07002420
2421 /* Do not cross zone boundaries */
Cody P Schafer108bcc92013-02-22 16:35:23 -08002422 if (!zone_spans_pfn(zone, start_pfn))
Mel Gormanc361be52007-10-16 01:25:51 -07002423 start_page = page;
Cody P Schafer108bcc92013-02-22 16:35:23 -08002424 if (!zone_spans_pfn(zone, end_pfn))
Mel Gormanc361be52007-10-16 01:25:51 -07002425 return 0;
2426
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002427 return move_freepages(zone, start_page, end_page, migratetype,
2428 num_movable);
Mel Gormanc361be52007-10-16 01:25:51 -07002429}
2430
Mel Gorman2f66a682009-09-21 17:02:31 -07002431static void change_pageblock_range(struct page *pageblock_page,
2432 int start_order, int migratetype)
2433{
2434 int nr_pageblocks = 1 << (start_order - pageblock_order);
2435
2436 while (nr_pageblocks--) {
2437 set_pageblock_migratetype(pageblock_page, migratetype);
2438 pageblock_page += pageblock_nr_pages;
2439 }
2440}
2441
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002442/*
Vlastimil Babka9c0415e2015-02-11 15:28:21 -08002443 * When we are falling back to another migratetype during allocation, try to
2444 * steal extra free pages from the same pageblocks to satisfy further
2445 * allocations, instead of polluting multiple pageblocks.
2446 *
2447 * If we are stealing a relatively large buddy page, it is likely there will
2448 * be more free pages in the pageblock, so try to steal them all. For
2449 * reclaimable and unmovable allocations, we steal regardless of page size,
2450 * as fragmentation caused by those allocations polluting movable pageblocks
2451 * is worse than movable allocations stealing from unmovable and reclaimable
2452 * pageblocks.
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002453 */
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002454static bool can_steal_fallback(unsigned int order, int start_mt)
2455{
2456 /*
2457 * Leaving this order check is intended, although there is
2458 * relaxed order check in next check. The reason is that
2459 * we can actually steal whole pageblock if this condition met,
2460 * but, below check doesn't guarantee it and that is just heuristic
2461 * so could be changed anytime.
2462 */
2463 if (order >= pageblock_order)
2464 return true;
2465
2466 if (order >= pageblock_order / 2 ||
2467 start_mt == MIGRATE_RECLAIMABLE ||
2468 start_mt == MIGRATE_UNMOVABLE ||
2469 page_group_by_mobility_disabled)
2470 return true;
2471
2472 return false;
2473}
2474
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002475static inline bool boost_watermark(struct zone *zone)
Mel Gorman1c308442018-12-28 00:35:52 -08002476{
2477 unsigned long max_boost;
2478
2479 if (!watermark_boost_factor)
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002480 return false;
Henry Willard14f69142020-05-07 18:36:27 -07002481 /*
2482 * Don't bother in zones that are unlikely to produce results.
2483 * On small machines, including kdump capture kernels running
2484 * in a small area, boosting the watermark can cause an out of
2485 * memory situation immediately.
2486 */
2487 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002488 return false;
Mel Gorman1c308442018-12-28 00:35:52 -08002489
2490 max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2491 watermark_boost_factor, 10000);
Mel Gorman94b33342019-02-20 22:19:49 -08002492
2493 /*
2494 * high watermark may be uninitialised if fragmentation occurs
2495 * very early in boot so do not boost. We do not fall
2496 * through and boost by pageblock_nr_pages as failing
2497 * allocations that early means that reclaim is not going
2498 * to help and it may even be impossible to reclaim the
2499 * boosted watermark resulting in a hang.
2500 */
2501 if (!max_boost)
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002502 return false;
Mel Gorman94b33342019-02-20 22:19:49 -08002503
Mel Gorman1c308442018-12-28 00:35:52 -08002504 max_boost = max(pageblock_nr_pages, max_boost);
2505
2506 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2507 max_boost);
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002508
2509 return true;
Mel Gorman1c308442018-12-28 00:35:52 -08002510}
2511
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002512/*
2513 * This function implements actual steal behaviour. If order is large enough,
2514 * we can steal whole pageblock. If not, we first move freepages in this
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002515 * pageblock to our migratetype and determine how many already-allocated pages
2516 * are there in the pageblock with a compatible migratetype. If at least half
2517 * of pages are free or compatible, we can change migratetype of the pageblock
2518 * itself, so pages freed in the future will be put on the correct free list.
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002519 */
2520static void steal_suitable_fallback(struct zone *zone, struct page *page,
Mel Gorman1c308442018-12-28 00:35:52 -08002521 unsigned int alloc_flags, int start_type, bool whole_block)
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002522{
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002523 unsigned int current_order = buddy_order(page);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002524 int free_pages, movable_pages, alike_pages;
2525 int old_block_type;
2526
2527 old_block_type = get_pageblock_migratetype(page);
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002528
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002529 /*
2530 * This can happen due to races and we want to prevent broken
2531 * highatomic accounting.
2532 */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002533 if (is_migrate_highatomic(old_block_type))
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002534 goto single_page;
2535
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002536 /* Take ownership for orders >= pageblock_order */
2537 if (current_order >= pageblock_order) {
2538 change_pageblock_range(page, current_order, start_type);
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002539 goto single_page;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002540 }
2541
Mel Gorman1c308442018-12-28 00:35:52 -08002542 /*
2543 * Boost watermarks to increase reclaim pressure to reduce the
2544 * likelihood of future fallbacks. Wake kswapd now as the node
2545 * may be balanced overall and kswapd will not wake naturally.
2546 */
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002547 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
Mel Gorman73444bc2019-01-08 15:23:39 -08002548 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
Mel Gorman1c308442018-12-28 00:35:52 -08002549
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002550 /* We are not allowed to try stealing from the whole block */
2551 if (!whole_block)
2552 goto single_page;
2553
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002554 free_pages = move_freepages_block(zone, page, start_type,
2555 &movable_pages);
2556 /*
2557 * Determine how many pages are compatible with our allocation.
2558 * For movable allocation, it's the number of movable pages which
2559 * we just obtained. For other types it's a bit more tricky.
2560 */
2561 if (start_type == MIGRATE_MOVABLE) {
2562 alike_pages = movable_pages;
2563 } else {
2564 /*
2565 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2566 * to MOVABLE pageblock, consider all non-movable pages as
2567 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2568 * vice versa, be conservative since we can't distinguish the
2569 * exact migratetype of non-movable pages.
2570 */
2571 if (old_block_type == MIGRATE_MOVABLE)
2572 alike_pages = pageblock_nr_pages
2573 - (free_pages + movable_pages);
2574 else
2575 alike_pages = 0;
2576 }
2577
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002578 /* moving whole block can fail due to zone boundary conditions */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002579 if (!free_pages)
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002580 goto single_page;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002581
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002582 /*
2583 * If a sufficient number of pages in the block are either free or of
2584 * comparable migratability as our allocation, claim the whole block.
2585 */
2586 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002587 page_group_by_mobility_disabled)
2588 set_pageblock_migratetype(page, start_type);
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002589
2590 return;
2591
2592single_page:
Alexander Duyck6ab01362020-04-06 20:04:49 -07002593 move_to_free_list(page, zone, current_order, start_type);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002594}
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002595
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002596/*
2597 * Check whether there is a suitable fallback freepage with requested order.
2598 * If only_stealable is true, this function returns fallback_mt only if
2599 * we can steal other freepages all together. This would help to reduce
2600 * fragmentation due to mixed migratetype pages in one pageblock.
2601 */
2602int find_suitable_fallback(struct free_area *area, unsigned int order,
2603 int migratetype, bool only_stealable, bool *can_steal)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002604{
2605 int i;
2606 int fallback_mt;
2607
2608 if (area->nr_free == 0)
2609 return -1;
2610
2611 *can_steal = false;
2612 for (i = 0;; i++) {
2613 fallback_mt = fallbacks[migratetype][i];
Mel Gorman974a7862015-11-06 16:28:34 -08002614 if (fallback_mt == MIGRATE_TYPES)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002615 break;
2616
Dan Williamsb03641a2019-05-14 15:41:32 -07002617 if (free_area_empty(area, fallback_mt))
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002618 continue;
2619
2620 if (can_steal_fallback(order, migratetype))
2621 *can_steal = true;
2622
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002623 if (!only_stealable)
2624 return fallback_mt;
2625
2626 if (*can_steal)
2627 return fallback_mt;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002628 }
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002629
2630 return -1;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002631}
2632
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002633/*
2634 * Reserve a pageblock for exclusive use of high-order atomic allocations if
2635 * there are no empty page blocks that contain a page with a suitable order
2636 */
2637static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2638 unsigned int alloc_order)
2639{
2640 int mt;
2641 unsigned long max_managed, flags;
2642
2643 /*
2644 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2645 * Check is race-prone but harmless.
2646 */
Arun KS9705bea2018-12-28 00:34:24 -08002647 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002648 if (zone->nr_reserved_highatomic >= max_managed)
2649 return;
2650
2651 spin_lock_irqsave(&zone->lock, flags);
2652
2653 /* Recheck the nr_reserved_highatomic limit under the lock */
2654 if (zone->nr_reserved_highatomic >= max_managed)
2655 goto out_unlock;
2656
2657 /* Yoink! */
2658 mt = get_pageblock_migratetype(page);
Xishi Qiua6ffdc02017-05-03 14:52:52 -07002659 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2660 && !is_migrate_cma(mt)) {
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002661 zone->nr_reserved_highatomic += pageblock_nr_pages;
2662 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002663 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002664 }
2665
2666out_unlock:
2667 spin_unlock_irqrestore(&zone->lock, flags);
2668}
2669
2670/*
2671 * Used when an allocation is about to fail under memory pressure. This
2672 * potentially hurts the reliability of high-order allocations when under
2673 * intense memory pressure but failed atomic allocations should be easier
2674 * to recover from than an OOM.
Minchan Kim29fac032016-12-12 16:42:14 -08002675 *
2676 * If @force is true, try to unreserve a pageblock even though highatomic
2677 * pageblock is exhausted.
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002678 */
Minchan Kim29fac032016-12-12 16:42:14 -08002679static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2680 bool force)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002681{
2682 struct zonelist *zonelist = ac->zonelist;
2683 unsigned long flags;
2684 struct zoneref *z;
2685 struct zone *zone;
2686 struct page *page;
2687 int order;
Minchan Kim04c87162016-12-12 16:42:11 -08002688 bool ret;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002689
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002690 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002691 ac->nodemask) {
Minchan Kim29fac032016-12-12 16:42:14 -08002692 /*
2693 * Preserve at least one pageblock unless memory pressure
2694 * is really high.
2695 */
2696 if (!force && zone->nr_reserved_highatomic <=
2697 pageblock_nr_pages)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002698 continue;
2699
2700 spin_lock_irqsave(&zone->lock, flags);
2701 for (order = 0; order < MAX_ORDER; order++) {
2702 struct free_area *area = &(zone->free_area[order]);
2703
Dan Williamsb03641a2019-05-14 15:41:32 -07002704 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
Geliang Tanga16601c2016-01-14 15:20:30 -08002705 if (!page)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002706 continue;
2707
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002708 /*
Minchan Kim4855e4a2016-12-12 16:42:08 -08002709 * In page freeing path, migratetype change is racy so
2710 * we can counter several free pages in a pageblock
2711 * in this loop althoug we changed the pageblock type
2712 * from highatomic to ac->migratetype. So we should
2713 * adjust the count once.
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002714 */
Xishi Qiua6ffdc02017-05-03 14:52:52 -07002715 if (is_migrate_highatomic_page(page)) {
Minchan Kim4855e4a2016-12-12 16:42:08 -08002716 /*
2717 * It should never happen but changes to
2718 * locking could inadvertently allow a per-cpu
2719 * drain to add pages to MIGRATE_HIGHATOMIC
2720 * while unreserving so be safe and watch for
2721 * underflows.
2722 */
2723 zone->nr_reserved_highatomic -= min(
2724 pageblock_nr_pages,
2725 zone->nr_reserved_highatomic);
2726 }
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002727
2728 /*
2729 * Convert to ac->migratetype and avoid the normal
2730 * pageblock stealing heuristics. Minimally, the caller
2731 * is doing the work and needs the pages. More
2732 * importantly, if the block was always converted to
2733 * MIGRATE_UNMOVABLE or another type then the number
2734 * of pageblocks that cannot be completely freed
2735 * may increase.
2736 */
2737 set_pageblock_migratetype(page, ac->migratetype);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002738 ret = move_freepages_block(zone, page, ac->migratetype,
2739 NULL);
Minchan Kim29fac032016-12-12 16:42:14 -08002740 if (ret) {
2741 spin_unlock_irqrestore(&zone->lock, flags);
2742 return ret;
2743 }
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002744 }
2745 spin_unlock_irqrestore(&zone->lock, flags);
2746 }
Minchan Kim04c87162016-12-12 16:42:11 -08002747
2748 return false;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002749}
2750
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002751/*
2752 * Try finding a free buddy page on the fallback list and put it on the free
2753 * list of requested migratetype, possibly along with other pages from the same
2754 * block, depending on fragmentation avoidance heuristics. Returns true if
2755 * fallback was found so that __rmqueue_smallest() can grab it.
Rasmus Villemoesb0025292017-07-10 15:49:26 -07002756 *
2757 * The use of signed ints for order and current_order is a deliberate
2758 * deviation from the rest of this file, to make the for loop
2759 * condition simpler.
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002760 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002761static __always_inline bool
Mel Gorman6bb15452018-12-28 00:35:41 -08002762__rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2763 unsigned int alloc_flags)
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002764{
Pintu Kumarb8af2942013-09-11 14:20:34 -07002765 struct free_area *area;
Rasmus Villemoesb0025292017-07-10 15:49:26 -07002766 int current_order;
Mel Gorman6bb15452018-12-28 00:35:41 -08002767 int min_order = order;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002768 struct page *page;
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002769 int fallback_mt;
2770 bool can_steal;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002771
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002772 /*
Mel Gorman6bb15452018-12-28 00:35:41 -08002773 * Do not steal pages from freelists belonging to other pageblocks
2774 * i.e. orders < pageblock_order. If there are no local zones free,
2775 * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2776 */
2777 if (alloc_flags & ALLOC_NOFRAGMENT)
2778 min_order = pageblock_order;
2779
2780 /*
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002781 * Find the largest available free page in the other list. This roughly
2782 * approximates finding the pageblock with the most free pages, which
2783 * would be too costly to do exactly.
2784 */
Mel Gorman6bb15452018-12-28 00:35:41 -08002785 for (current_order = MAX_ORDER - 1; current_order >= min_order;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002786 --current_order) {
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002787 area = &(zone->free_area[current_order]);
2788 fallback_mt = find_suitable_fallback(area, current_order,
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002789 start_migratetype, false, &can_steal);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002790 if (fallback_mt == -1)
2791 continue;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002792
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002793 /*
2794 * We cannot steal all free pages from the pageblock and the
2795 * requested migratetype is movable. In that case it's better to
2796 * steal and split the smallest available page instead of the
2797 * largest available page, because even if the next movable
2798 * allocation falls back into a different pageblock than this
2799 * one, it won't cause permanent fragmentation.
2800 */
2801 if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2802 && current_order > order)
2803 goto find_smallest;
Mel Gormane0104872007-10-16 01:25:53 -07002804
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002805 goto do_steal;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002806 }
2807
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002808 return false;
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002809
2810find_smallest:
2811 for (current_order = order; current_order < MAX_ORDER;
2812 current_order++) {
2813 area = &(zone->free_area[current_order]);
2814 fallback_mt = find_suitable_fallback(area, current_order,
2815 start_migratetype, false, &can_steal);
2816 if (fallback_mt != -1)
2817 break;
2818 }
2819
2820 /*
2821 * This should not happen - we already found a suitable fallback
2822 * when looking for the largest page.
2823 */
2824 VM_BUG_ON(current_order == MAX_ORDER);
2825
2826do_steal:
Dan Williamsb03641a2019-05-14 15:41:32 -07002827 page = get_page_from_free_area(area, fallback_mt);
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002828
Mel Gorman1c308442018-12-28 00:35:52 -08002829 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2830 can_steal);
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002831
2832 trace_mm_page_alloc_extfrag(page, order, current_order,
2833 start_migratetype, fallback_mt);
2834
2835 return true;
2836
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002837}
2838
Mel Gorman56fd56b2007-10-16 01:25:58 -07002839/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002840 * Do the hard work of removing an element from the buddy allocator.
2841 * Call me with the zone->lock already held.
2842 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002843static __always_inline struct page *
Mel Gorman6bb15452018-12-28 00:35:41 -08002844__rmqueue(struct zone *zone, unsigned int order, int migratetype,
2845 unsigned int alloc_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002846{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002847 struct page *page;
2848
Hailong liuc11f7742021-01-12 15:49:08 -08002849 if (IS_ENABLED(CONFIG_CMA)) {
2850 /*
2851 * Balance movable allocations between regular and CMA areas by
2852 * allocating from CMA when over half of the zone's free memory
2853 * is in the CMA area.
2854 */
2855 if (alloc_flags & ALLOC_CMA &&
2856 zone_page_state(zone, NR_FREE_CMA_PAGES) >
2857 zone_page_state(zone, NR_FREE_PAGES) / 2) {
2858 page = __rmqueue_cma_fallback(zone, order);
2859 if (page)
2860 goto out;
2861 }
Roman Gushchin16867662020-06-03 15:58:42 -07002862 }
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002863retry:
Mel Gorman56fd56b2007-10-16 01:25:58 -07002864 page = __rmqueue_smallest(zone, order, migratetype);
Mel Gorman974a7862015-11-06 16:28:34 -08002865 if (unlikely(!page)) {
Joonsoo Kim8510e692020-08-06 23:26:04 -07002866 if (alloc_flags & ALLOC_CMA)
Joonsoo Kimdc676472015-04-14 15:45:15 -07002867 page = __rmqueue_cma_fallback(zone, order);
2868
Mel Gorman6bb15452018-12-28 00:35:41 -08002869 if (!page && __rmqueue_fallback(zone, order, migratetype,
2870 alloc_flags))
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002871 goto retry;
Mel Gorman728ec982009-06-16 15:32:04 -07002872 }
Hailong liuc11f7742021-01-12 15:49:08 -08002873out:
2874 if (page)
2875 trace_mm_page_alloc_zone_locked(page, order, migratetype);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002876 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002877}
2878
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002879/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002880 * Obtain a specified number of elements from the buddy allocator, all under
2881 * a single hold of the lock, for efficiency. Add them to the supplied list.
2882 * Returns the number of new pages which were placed at *list.
2883 */
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002884static int rmqueue_bulk(struct zone *zone, unsigned int order,
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002885 unsigned long count, struct list_head *list,
Mel Gorman6bb15452018-12-28 00:35:41 -08002886 int migratetype, unsigned int alloc_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002887{
Mel Gormana6de7342016-12-12 16:44:41 -08002888 int i, alloced = 0;
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002889
Mel Gormand34b0732017-04-20 14:37:43 -07002890 spin_lock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002891 for (i = 0; i < count; ++i) {
Mel Gorman6bb15452018-12-28 00:35:41 -08002892 struct page *page = __rmqueue(zone, order, migratetype,
2893 alloc_flags);
Nick Piggin085cc7d52006-01-06 00:11:01 -08002894 if (unlikely(page == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002895 break;
Mel Gorman81eabcb2007-12-17 16:20:05 -08002896
Mel Gorman479f8542016-05-19 17:14:35 -07002897 if (unlikely(check_pcp_refill(page)))
2898 continue;
2899
Mel Gorman81eabcb2007-12-17 16:20:05 -08002900 /*
Vlastimil Babka0fac3ba2017-11-15 17:38:07 -08002901 * Split buddy pages returned by expand() are received here in
2902 * physical page order. The page is added to the tail of
2903 * caller's list. From the callers perspective, the linked list
2904 * is ordered by page number under some conditions. This is
2905 * useful for IO devices that can forward direction from the
2906 * head, thus also in the physical page order. This is useful
2907 * for IO devices that can merge IO requests if the physical
2908 * pages are ordered properly.
Mel Gorman81eabcb2007-12-17 16:20:05 -08002909 */
Vlastimil Babka0fac3ba2017-11-15 17:38:07 -08002910 list_add_tail(&page->lru, list);
Mel Gormana6de7342016-12-12 16:44:41 -08002911 alloced++;
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002912 if (is_migrate_cma(get_pcppage_migratetype(page)))
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07002913 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2914 -(1 << order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002915 }
Mel Gormana6de7342016-12-12 16:44:41 -08002916
2917 /*
2918 * i pages were removed from the buddy list even if some leak due
2919 * to check_pcp_refill failing so adjust NR_FREE_PAGES based
2920 * on i. Do not confuse with 'alloced' which is the number of
2921 * pages added to the pcp list.
2922 */
Mel Gormanf2260e62009-06-16 15:32:13 -07002923 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
Mel Gormand34b0732017-04-20 14:37:43 -07002924 spin_unlock(&zone->lock);
Mel Gormana6de7342016-12-12 16:44:41 -08002925 return alloced;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002926}
2927
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002928#ifdef CONFIG_NUMA
Christoph Lameter8fce4d82006-03-09 17:33:54 -08002929/*
Christoph Lameter4037d452007-05-09 02:35:14 -07002930 * Called from the vmstat counter updater to drain pagesets of this
2931 * currently executing processor on remote nodes after they have
2932 * expired.
2933 *
Christoph Lameter879336c2006-03-22 00:09:08 -08002934 * Note that this function must be called with the thread pinned to
2935 * a single processor.
Christoph Lameter8fce4d82006-03-09 17:33:54 -08002936 */
Christoph Lameter4037d452007-05-09 02:35:14 -07002937void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002938{
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002939 unsigned long flags;
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07002940 int to_drain, batch;
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002941
Christoph Lameter4037d452007-05-09 02:35:14 -07002942 local_irq_save(flags);
Jason Low4db0c3c2015-04-15 16:14:08 -07002943 batch = READ_ONCE(pcp->batch);
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07002944 to_drain = min(pcp->count, batch);
Aaron Lu77ba9062018-04-05 16:24:06 -07002945 if (to_drain > 0)
KOSAKI Motohiro2a135152012-07-31 16:42:53 -07002946 free_pcppages_bulk(zone, to_drain, pcp);
Christoph Lameter4037d452007-05-09 02:35:14 -07002947 local_irq_restore(flags);
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002948}
2949#endif
2950
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002951/*
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002952 * Drain pcplists of the indicated processor and zone.
2953 *
2954 * The processor must either be the current processor and the
2955 * thread pinned to the current processor or a processor that
2956 * is not online.
2957 */
2958static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2959{
2960 unsigned long flags;
2961 struct per_cpu_pageset *pset;
2962 struct per_cpu_pages *pcp;
2963
2964 local_irq_save(flags);
2965 pset = per_cpu_ptr(zone->pageset, cpu);
2966
2967 pcp = &pset->pcp;
Aaron Lu77ba9062018-04-05 16:24:06 -07002968 if (pcp->count)
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002969 free_pcppages_bulk(zone, pcp->count, pcp);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002970 local_irq_restore(flags);
2971}
2972
2973/*
2974 * Drain pcplists of all zones on the indicated processor.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002975 *
2976 * The processor must either be the current processor and the
2977 * thread pinned to the current processor or a processor that
2978 * is not online.
2979 */
2980static void drain_pages(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002981{
2982 struct zone *zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002983
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07002984 for_each_populated_zone(zone) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002985 drain_pages_zone(cpu, zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002986 }
2987}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002988
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002989/*
2990 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002991 *
2992 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
2993 * the single zone's pages.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002994 */
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002995void drain_local_pages(struct zone *zone)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002996{
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002997 int cpu = smp_processor_id();
2998
2999 if (zone)
3000 drain_pages_zone(cpu, zone);
3001 else
3002 drain_pages(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003003}
3004
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003005static void drain_local_pages_wq(struct work_struct *work)
3006{
Wei Yangd9367bd2018-12-28 00:38:58 -08003007 struct pcpu_drain *drain;
3008
3009 drain = container_of(work, struct pcpu_drain, work);
3010
Michal Hockoa459eeb2017-02-24 14:56:35 -08003011 /*
3012 * drain_all_pages doesn't use proper cpu hotplug protection so
3013 * we can race with cpu offline when the WQ can move this from
3014 * a cpu pinned worker to an unbound one. We can operate on a different
3015 * cpu which is allright but we also have to make sure to not move to
3016 * a different one.
3017 */
3018 preempt_disable();
Wei Yangd9367bd2018-12-28 00:38:58 -08003019 drain_local_pages(drain->zone);
Michal Hockoa459eeb2017-02-24 14:56:35 -08003020 preempt_enable();
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003021}
3022
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003023/*
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003024 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
3025 *
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003026 * When zone parameter is non-NULL, spill just the single zone's pages.
3027 *
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003028 * Note that this can be extremely slow as the draining happens in a workqueue.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003029 */
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003030void drain_all_pages(struct zone *zone)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003031{
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003032 int cpu;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003033
3034 /*
3035 * Allocate in the BSS so we wont require allocation in
3036 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
3037 */
3038 static cpumask_t cpus_with_pcps;
3039
Michal Hockoce612872017-04-07 16:05:05 -07003040 /*
3041 * Make sure nobody triggers this path before mm_percpu_wq is fully
3042 * initialized.
3043 */
3044 if (WARN_ON_ONCE(!mm_percpu_wq))
3045 return;
3046
Mel Gormanbd233f52017-02-24 14:56:56 -08003047 /*
3048 * Do not drain if one is already in progress unless it's specific to
3049 * a zone. Such callers are primarily CMA and memory hotplug and need
3050 * the drain to be complete when the call returns.
3051 */
3052 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
3053 if (!zone)
3054 return;
3055 mutex_lock(&pcpu_drain_mutex);
3056 }
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003057
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003058 /*
3059 * We don't care about racing with CPU hotplug event
3060 * as offline notification will cause the notified
3061 * cpu to drain that CPU pcps and on_each_cpu_mask
3062 * disables preemption as part of its processing
3063 */
3064 for_each_online_cpu(cpu) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003065 struct per_cpu_pageset *pcp;
3066 struct zone *z;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003067 bool has_pcps = false;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003068
3069 if (zone) {
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003070 pcp = per_cpu_ptr(zone->pageset, cpu);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003071 if (pcp->pcp.count)
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003072 has_pcps = true;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003073 } else {
3074 for_each_populated_zone(z) {
3075 pcp = per_cpu_ptr(z->pageset, cpu);
3076 if (pcp->pcp.count) {
3077 has_pcps = true;
3078 break;
3079 }
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003080 }
3081 }
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003082
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003083 if (has_pcps)
3084 cpumask_set_cpu(cpu, &cpus_with_pcps);
3085 else
3086 cpumask_clear_cpu(cpu, &cpus_with_pcps);
3087 }
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003088
Mel Gormanbd233f52017-02-24 14:56:56 -08003089 for_each_cpu(cpu, &cpus_with_pcps) {
Wei Yangd9367bd2018-12-28 00:38:58 -08003090 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
3091
3092 drain->zone = zone;
3093 INIT_WORK(&drain->work, drain_local_pages_wq);
3094 queue_work_on(cpu, mm_percpu_wq, &drain->work);
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003095 }
Mel Gormanbd233f52017-02-24 14:56:56 -08003096 for_each_cpu(cpu, &cpus_with_pcps)
Wei Yangd9367bd2018-12-28 00:38:58 -08003097 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
Mel Gormanbd233f52017-02-24 14:56:56 -08003098
3099 mutex_unlock(&pcpu_drain_mutex);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003100}
3101
Rafael J. Wysocki296699d2007-07-29 23:27:18 +02003102#ifdef CONFIG_HIBERNATION
Linus Torvalds1da177e2005-04-16 15:20:36 -07003103
Chen Yu556b9692017-08-25 15:55:30 -07003104/*
3105 * Touch the watchdog for every WD_PAGE_COUNT pages.
3106 */
3107#define WD_PAGE_COUNT (128*1024)
3108
Linus Torvalds1da177e2005-04-16 15:20:36 -07003109void mark_free_pages(struct zone *zone)
3110{
Chen Yu556b9692017-08-25 15:55:30 -07003111 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003112 unsigned long flags;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003113 unsigned int order, t;
Geliang Tang86760a22016-01-14 15:20:33 -08003114 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003115
Xishi Qiu8080fc02013-09-11 14:21:45 -07003116 if (zone_is_empty(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003117 return;
3118
3119 spin_lock_irqsave(&zone->lock, flags);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003120
Cody P Schafer108bcc92013-02-22 16:35:23 -08003121 max_zone_pfn = zone_end_pfn(zone);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003122 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
3123 if (pfn_valid(pfn)) {
Geliang Tang86760a22016-01-14 15:20:33 -08003124 page = pfn_to_page(pfn);
Joonsoo Kimba6b0972016-05-19 17:12:16 -07003125
Chen Yu556b9692017-08-25 15:55:30 -07003126 if (!--page_count) {
3127 touch_nmi_watchdog();
3128 page_count = WD_PAGE_COUNT;
3129 }
3130
Joonsoo Kimba6b0972016-05-19 17:12:16 -07003131 if (page_zone(page) != zone)
3132 continue;
3133
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07003134 if (!swsusp_page_is_forbidden(page))
3135 swsusp_unset_page_free(page);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003136 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003137
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003138 for_each_migratetype_order(order, t) {
Geliang Tang86760a22016-01-14 15:20:33 -08003139 list_for_each_entry(page,
3140 &zone->free_area[order].free_list[t], lru) {
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003141 unsigned long i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003142
Geliang Tang86760a22016-01-14 15:20:33 -08003143 pfn = page_to_pfn(page);
Chen Yu556b9692017-08-25 15:55:30 -07003144 for (i = 0; i < (1UL << order); i++) {
3145 if (!--page_count) {
3146 touch_nmi_watchdog();
3147 page_count = WD_PAGE_COUNT;
3148 }
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07003149 swsusp_set_page_free(pfn_to_page(pfn + i));
Chen Yu556b9692017-08-25 15:55:30 -07003150 }
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003151 }
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003152 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003153 spin_unlock_irqrestore(&zone->lock, flags);
3154}
Mel Gormane2c55dc2007-10-16 01:25:50 -07003155#endif /* CONFIG_PM */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003156
Mel Gorman2d4894b2017-11-15 17:37:59 -08003157static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158{
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003159 int migratetype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003160
Mel Gorman4db75482016-05-19 17:14:32 -07003161 if (!free_pcp_prepare(page))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003162 return false;
Hugh Dickins689bceb2005-11-21 21:32:20 -08003163
Mel Gormandc4b0ca2014-06-04 16:10:17 -07003164 migratetype = get_pfnblock_migratetype(page, pfn);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07003165 set_pcppage_migratetype(page, migratetype);
Mel Gorman9cca35d42017-11-15 17:37:37 -08003166 return true;
3167}
3168
Mel Gorman2d4894b2017-11-15 17:37:59 -08003169static void free_unref_page_commit(struct page *page, unsigned long pfn)
Mel Gorman9cca35d42017-11-15 17:37:37 -08003170{
3171 struct zone *zone = page_zone(page);
3172 struct per_cpu_pages *pcp;
3173 int migratetype;
3174
3175 migratetype = get_pcppage_migratetype(page);
Mel Gormand34b0732017-04-20 14:37:43 -07003176 __count_vm_event(PGFREE);
Mel Gormanda456f12009-06-16 15:32:08 -07003177
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003178 /*
3179 * We only track unmovable, reclaimable and movable on pcp lists.
3180 * Free ISOLATE pages back to the allocator because they are being
Xishi Qiua6ffdc02017-05-03 14:52:52 -07003181 * offlined but treat HIGHATOMIC as movable pages so we can get those
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003182 * areas back if necessary. Otherwise, we may have to free
3183 * excessively into the page allocator
3184 */
3185 if (migratetype >= MIGRATE_PCPTYPES) {
Minchan Kim194159f2013-02-22 16:33:58 -08003186 if (unlikely(is_migrate_isolate(migratetype))) {
David Hildenbrand7fef4312020-10-15 20:09:35 -07003187 free_one_page(zone, page, pfn, 0, migratetype,
3188 FPI_NONE);
Mel Gorman9cca35d42017-11-15 17:37:37 -08003189 return;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003190 }
3191 migratetype = MIGRATE_MOVABLE;
3192 }
3193
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003194 pcp = &this_cpu_ptr(zone->pageset)->pcp;
Mel Gorman2d4894b2017-11-15 17:37:59 -08003195 list_add(&page->lru, &pcp->lists[migratetype]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003196 pcp->count++;
Nick Piggin48db57f2006-01-08 01:00:42 -08003197 if (pcp->count >= pcp->high) {
Jason Low4db0c3c2015-04-15 16:14:08 -07003198 unsigned long batch = READ_ONCE(pcp->batch);
Cody P Schafer998d39cb2013-07-03 15:01:32 -07003199 free_pcppages_bulk(zone, batch, pcp);
Nick Piggin48db57f2006-01-08 01:00:42 -08003200 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003201}
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003202
Mel Gorman9cca35d42017-11-15 17:37:37 -08003203/*
3204 * Free a 0-order page
Mel Gorman9cca35d42017-11-15 17:37:37 -08003205 */
Mel Gorman2d4894b2017-11-15 17:37:59 -08003206void free_unref_page(struct page *page)
Mel Gorman9cca35d42017-11-15 17:37:37 -08003207{
3208 unsigned long flags;
3209 unsigned long pfn = page_to_pfn(page);
3210
Mel Gorman2d4894b2017-11-15 17:37:59 -08003211 if (!free_unref_page_prepare(page, pfn))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003212 return;
3213
3214 local_irq_save(flags);
Mel Gorman2d4894b2017-11-15 17:37:59 -08003215 free_unref_page_commit(page, pfn);
Mel Gormand34b0732017-04-20 14:37:43 -07003216 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003217}
3218
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003219/*
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003220 * Free a list of 0-order pages
3221 */
Mel Gorman2d4894b2017-11-15 17:37:59 -08003222void free_unref_page_list(struct list_head *list)
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003223{
3224 struct page *page, *next;
Mel Gorman9cca35d42017-11-15 17:37:37 -08003225 unsigned long flags, pfn;
Lucas Stachc24ad772017-12-14 15:32:55 -08003226 int batch_count = 0;
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003227
Mel Gorman9cca35d42017-11-15 17:37:37 -08003228 /* Prepare pages for freeing */
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003229 list_for_each_entry_safe(page, next, list, lru) {
Mel Gorman9cca35d42017-11-15 17:37:37 -08003230 pfn = page_to_pfn(page);
Mel Gorman2d4894b2017-11-15 17:37:59 -08003231 if (!free_unref_page_prepare(page, pfn))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003232 list_del(&page->lru);
3233 set_page_private(page, pfn);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003234 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003235
3236 local_irq_save(flags);
3237 list_for_each_entry_safe(page, next, list, lru) {
3238 unsigned long pfn = page_private(page);
3239
3240 set_page_private(page, 0);
Mel Gorman2d4894b2017-11-15 17:37:59 -08003241 trace_mm_page_free_batched(page);
3242 free_unref_page_commit(page, pfn);
Lucas Stachc24ad772017-12-14 15:32:55 -08003243
3244 /*
3245 * Guard against excessive IRQ disabled times when we get
3246 * a large list of pages to free.
3247 */
3248 if (++batch_count == SWAP_CLUSTER_MAX) {
3249 local_irq_restore(flags);
3250 batch_count = 0;
3251 local_irq_save(flags);
3252 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003253 }
3254 local_irq_restore(flags);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003255}
3256
3257/*
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003258 * split_page takes a non-compound higher-order page, and splits it into
3259 * n (1<<order) sub-pages: page[0..n]
3260 * Each sub-page must be freed individually.
3261 *
3262 * Note: this is probably too low level an operation for use in drivers.
3263 * Please consult with lkml before using this in your driver.
3264 */
3265void split_page(struct page *page, unsigned int order)
3266{
3267 int i;
3268
Sasha Levin309381fea2014-01-23 15:52:54 -08003269 VM_BUG_ON_PAGE(PageCompound(page), page);
3270 VM_BUG_ON_PAGE(!page_count(page), page);
Vegard Nossumb1eeab62008-11-25 16:55:53 +01003271
Joonsoo Kima9627bc2016-07-26 15:23:49 -07003272 for (i = 1; i < (1 << order); i++)
Nick Piggin7835e982006-03-22 00:08:40 -08003273 set_page_refcounted(page + i);
Matthew Wilcox (Oracle)8fb156c2020-10-15 20:05:29 -07003274 split_page_owner(page, 1 << order);
Zhou Guanghuiefb12c02021-03-12 21:08:33 -08003275 split_page_memcg(page, 1 << order);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003276}
K. Y. Srinivasan5853ff22013-03-25 15:47:38 -07003277EXPORT_SYMBOL_GPL(split_page);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003278
Joonsoo Kim3c605092014-11-13 15:19:21 -08003279int __isolate_free_page(struct page *page, unsigned int order)
Mel Gorman748446b2010-05-24 14:32:27 -07003280{
Mel Gorman748446b2010-05-24 14:32:27 -07003281 unsigned long watermark;
3282 struct zone *zone;
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07003283 int mt;
Mel Gorman748446b2010-05-24 14:32:27 -07003284
3285 BUG_ON(!PageBuddy(page));
3286
3287 zone = page_zone(page);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003288 mt = get_pageblock_migratetype(page);
Mel Gorman748446b2010-05-24 14:32:27 -07003289
Minchan Kim194159f2013-02-22 16:33:58 -08003290 if (!is_migrate_isolate(mt)) {
Vlastimil Babka8348faf2016-10-07 16:58:00 -07003291 /*
3292 * Obey watermarks as if the page was being allocated. We can
3293 * emulate a high-order watermark check with a raised order-0
3294 * watermark, because we already know our high-order page
3295 * exists.
3296 */
Mel Gormanfd1444b2019-03-05 15:44:50 -08003297 watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003298 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003299 return 0;
3300
Mel Gorman8fb74b92013-01-11 14:32:16 -08003301 __mod_zone_freepage_state(zone, -(1UL << order), mt);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003302 }
Mel Gorman748446b2010-05-24 14:32:27 -07003303
3304 /* Remove page from free list */
Dan Williamsb03641a2019-05-14 15:41:32 -07003305
Alexander Duyck6ab01362020-04-06 20:04:49 -07003306 del_page_from_free_list(page, zone, order);
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07003307
zhong jiang400bc7f2016-07-28 15:45:07 -07003308 /*
3309 * Set the pageblock if the isolated page is at least half of a
3310 * pageblock
3311 */
Mel Gorman748446b2010-05-24 14:32:27 -07003312 if (order >= pageblock_order - 1) {
3313 struct page *endpage = page + (1 << order) - 1;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01003314 for (; page < endpage; page += pageblock_nr_pages) {
3315 int mt = get_pageblock_migratetype(page);
Minchan Kim88ed3652016-12-12 16:42:05 -08003316 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
Xishi Qiua6ffdc02017-05-03 14:52:52 -07003317 && !is_migrate_highatomic(mt))
Michal Nazarewicz47118af2011-12-29 13:09:50 +01003318 set_pageblock_migratetype(page,
3319 MIGRATE_MOVABLE);
3320 }
Mel Gorman748446b2010-05-24 14:32:27 -07003321 }
3322
Joonsoo Kimf3a14ce2015-07-17 16:24:15 -07003323
Mel Gorman8fb74b92013-01-11 14:32:16 -08003324 return 1UL << order;
Mel Gorman1fb3f8c2012-10-08 16:29:12 -07003325}
3326
Alexander Duyck624f58d2020-04-06 20:04:53 -07003327/**
3328 * __putback_isolated_page - Return a now-isolated page back where we got it
3329 * @page: Page that was isolated
3330 * @order: Order of the isolated page
Randy Dunlape6a0a7a2020-04-10 14:32:29 -07003331 * @mt: The page's pageblock's migratetype
Alexander Duyck624f58d2020-04-06 20:04:53 -07003332 *
3333 * This function is meant to return a page pulled from the free lists via
3334 * __isolate_free_page back to the free lists they were pulled from.
3335 */
3336void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3337{
3338 struct zone *zone = page_zone(page);
3339
3340 /* zone lock should be held when this function is called */
3341 lockdep_assert_held(&zone->lock);
3342
3343 /* Return isolated page to tail of freelist. */
David Hildenbrandf04a5d52020-10-15 20:09:20 -07003344 __free_one_page(page, page_to_pfn(page), zone, order, mt,
David Hildenbrand47b6a24a22020-10-15 20:09:26 -07003345 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
Alexander Duyck624f58d2020-04-06 20:04:53 -07003346}
3347
Mel Gorman1fb3f8c2012-10-08 16:29:12 -07003348/*
Mel Gorman060e7412016-05-19 17:13:27 -07003349 * Update NUMA hit/miss statistics
3350 *
3351 * Must be called with interrupts disabled.
Mel Gorman060e7412016-05-19 17:13:27 -07003352 */
Michal Hocko41b61672017-01-10 16:57:42 -08003353static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
Mel Gorman060e7412016-05-19 17:13:27 -07003354{
3355#ifdef CONFIG_NUMA
Kemi Wang3a321d22017-09-08 16:12:48 -07003356 enum numa_stat_item local_stat = NUMA_LOCAL;
Mel Gorman060e7412016-05-19 17:13:27 -07003357
Kemi Wang45180852017-11-15 17:38:22 -08003358 /* skip numa counters update if numa stats is disabled */
3359 if (!static_branch_likely(&vm_numa_stat_key))
3360 return;
3361
Pavel Tatashinc1093b72018-08-21 21:53:32 -07003362 if (zone_to_nid(z) != numa_node_id())
Mel Gorman060e7412016-05-19 17:13:27 -07003363 local_stat = NUMA_OTHER;
Mel Gorman060e7412016-05-19 17:13:27 -07003364
Pavel Tatashinc1093b72018-08-21 21:53:32 -07003365 if (zone_to_nid(z) == zone_to_nid(preferred_zone))
Kemi Wang3a321d22017-09-08 16:12:48 -07003366 __inc_numa_state(z, NUMA_HIT);
Michal Hocko2df26632017-01-10 16:57:39 -08003367 else {
Kemi Wang3a321d22017-09-08 16:12:48 -07003368 __inc_numa_state(z, NUMA_MISS);
3369 __inc_numa_state(preferred_zone, NUMA_FOREIGN);
Mel Gorman060e7412016-05-19 17:13:27 -07003370 }
Kemi Wang3a321d22017-09-08 16:12:48 -07003371 __inc_numa_state(z, local_stat);
Mel Gorman060e7412016-05-19 17:13:27 -07003372#endif
3373}
3374
Mel Gorman066b2392017-02-24 14:56:26 -08003375/* Remove page from the per-cpu list, caller must protect the list */
3376static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
Mel Gorman6bb15452018-12-28 00:35:41 -08003377 unsigned int alloc_flags,
Mel Gorman453f85d2017-11-15 17:38:03 -08003378 struct per_cpu_pages *pcp,
Mel Gorman066b2392017-02-24 14:56:26 -08003379 struct list_head *list)
3380{
3381 struct page *page;
3382
3383 do {
3384 if (list_empty(list)) {
3385 pcp->count += rmqueue_bulk(zone, 0,
3386 pcp->batch, list,
Mel Gorman6bb15452018-12-28 00:35:41 -08003387 migratetype, alloc_flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003388 if (unlikely(list_empty(list)))
3389 return NULL;
3390 }
3391
Mel Gorman453f85d2017-11-15 17:38:03 -08003392 page = list_first_entry(list, struct page, lru);
Mel Gorman066b2392017-02-24 14:56:26 -08003393 list_del(&page->lru);
3394 pcp->count--;
3395 } while (check_new_pcp(page));
3396
3397 return page;
3398}
3399
3400/* Lock and remove page from the per-cpu list */
3401static struct page *rmqueue_pcplist(struct zone *preferred_zone,
Yafang Shao1c52e6d2019-05-13 17:22:40 -07003402 struct zone *zone, gfp_t gfp_flags,
3403 int migratetype, unsigned int alloc_flags)
Mel Gorman066b2392017-02-24 14:56:26 -08003404{
3405 struct per_cpu_pages *pcp;
3406 struct list_head *list;
Mel Gorman066b2392017-02-24 14:56:26 -08003407 struct page *page;
Mel Gormand34b0732017-04-20 14:37:43 -07003408 unsigned long flags;
Mel Gorman066b2392017-02-24 14:56:26 -08003409
Mel Gormand34b0732017-04-20 14:37:43 -07003410 local_irq_save(flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003411 pcp = &this_cpu_ptr(zone->pageset)->pcp;
3412 list = &pcp->lists[migratetype];
Mel Gorman6bb15452018-12-28 00:35:41 -08003413 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list);
Mel Gorman066b2392017-02-24 14:56:26 -08003414 if (page) {
Yafang Shao1c52e6d2019-05-13 17:22:40 -07003415 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
Mel Gorman066b2392017-02-24 14:56:26 -08003416 zone_statistics(preferred_zone, zone);
3417 }
Mel Gormand34b0732017-04-20 14:37:43 -07003418 local_irq_restore(flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003419 return page;
3420}
3421
Mel Gorman060e7412016-05-19 17:13:27 -07003422/*
Vlastimil Babka75379192015-02-11 15:25:38 -08003423 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003424 */
Mel Gorman0a15c3e2009-06-16 15:32:05 -07003425static inline
Mel Gorman066b2392017-02-24 14:56:26 -08003426struct page *rmqueue(struct zone *preferred_zone,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003427 struct zone *zone, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07003428 gfp_t gfp_flags, unsigned int alloc_flags,
3429 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003430{
3431 unsigned long flags;
Hugh Dickins689bceb2005-11-21 21:32:20 -08003432 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003433
Mel Gormand34b0732017-04-20 14:37:43 -07003434 if (likely(order == 0)) {
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003435 /*
3436 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and
3437 * we need to skip it when CMA area isn't allowed.
3438 */
3439 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
3440 migratetype != MIGRATE_MOVABLE) {
3441 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
Yafang Shao1c52e6d2019-05-13 17:22:40 -07003442 migratetype, alloc_flags);
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003443 goto out;
3444 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003445 }
3446
Mel Gorman066b2392017-02-24 14:56:26 -08003447 /*
3448 * We most definitely don't want callers attempting to
3449 * allocate greater than order-1 page units with __GFP_NOFAIL.
3450 */
3451 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3452 spin_lock_irqsave(&zone->lock, flags);
3453
3454 do {
3455 page = NULL;
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003456 /*
3457 * order-0 request can reach here when the pcplist is skipped
3458 * due to non-CMA allocation context. HIGHATOMIC area is
3459 * reserved for high-order atomic allocation, so order-0
3460 * request should skip it.
3461 */
3462 if (order > 0 && alloc_flags & ALLOC_HARDER) {
Mel Gorman066b2392017-02-24 14:56:26 -08003463 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3464 if (page)
3465 trace_mm_page_alloc_zone_locked(page, order, migratetype);
3466 }
3467 if (!page)
Mel Gorman6bb15452018-12-28 00:35:41 -08003468 page = __rmqueue(zone, order, migratetype, alloc_flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003469 } while (page && check_new_pages(page, order));
3470 spin_unlock(&zone->lock);
3471 if (!page)
3472 goto failed;
3473 __mod_zone_freepage_state(zone, -(1 << order),
3474 get_pcppage_migratetype(page));
3475
Mel Gorman16709d12016-07-28 15:46:56 -07003476 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
Michal Hocko41b61672017-01-10 16:57:42 -08003477 zone_statistics(preferred_zone, zone);
Nick Piggina74609f2006-01-06 00:11:20 -08003478 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003479
Mel Gorman066b2392017-02-24 14:56:26 -08003480out:
Mel Gorman73444bc2019-01-08 15:23:39 -08003481 /* Separate test+clear to avoid unnecessary atomics */
3482 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3483 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3484 wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3485 }
3486
Mel Gorman066b2392017-02-24 14:56:26 -08003487 VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003488 return page;
Nick Piggina74609f2006-01-06 00:11:20 -08003489
3490failed:
3491 local_irq_restore(flags);
Nick Piggina74609f2006-01-06 00:11:20 -08003492 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003493}
3494
Akinobu Mita933e3122006-12-08 02:39:45 -08003495#ifdef CONFIG_FAIL_PAGE_ALLOC
3496
Akinobu Mitab2588c42011-07-26 16:09:03 -07003497static struct {
Akinobu Mita933e3122006-12-08 02:39:45 -08003498 struct fault_attr attr;
3499
Viresh Kumar621a5f72015-09-26 15:04:07 -07003500 bool ignore_gfp_highmem;
Mel Gorman71baba42015-11-06 16:28:28 -08003501 bool ignore_gfp_reclaim;
Akinobu Mita54114992007-07-15 23:40:23 -07003502 u32 min_order;
Akinobu Mita933e3122006-12-08 02:39:45 -08003503} fail_page_alloc = {
3504 .attr = FAULT_ATTR_INITIALIZER,
Mel Gorman71baba42015-11-06 16:28:28 -08003505 .ignore_gfp_reclaim = true,
Viresh Kumar621a5f72015-09-26 15:04:07 -07003506 .ignore_gfp_highmem = true,
Akinobu Mita54114992007-07-15 23:40:23 -07003507 .min_order = 1,
Akinobu Mita933e3122006-12-08 02:39:45 -08003508};
3509
3510static int __init setup_fail_page_alloc(char *str)
3511{
3512 return setup_fault_attr(&fail_page_alloc.attr, str);
3513}
3514__setup("fail_page_alloc=", setup_fail_page_alloc);
3515
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003516static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08003517{
Akinobu Mita54114992007-07-15 23:40:23 -07003518 if (order < fail_page_alloc.min_order)
Gavin Shandeaf3862012-07-31 16:41:51 -07003519 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003520 if (gfp_mask & __GFP_NOFAIL)
Gavin Shandeaf3862012-07-31 16:41:51 -07003521 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003522 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
Gavin Shandeaf3862012-07-31 16:41:51 -07003523 return false;
Mel Gorman71baba42015-11-06 16:28:28 -08003524 if (fail_page_alloc.ignore_gfp_reclaim &&
3525 (gfp_mask & __GFP_DIRECT_RECLAIM))
Gavin Shandeaf3862012-07-31 16:41:51 -07003526 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003527
3528 return should_fail(&fail_page_alloc.attr, 1 << order);
3529}
3530
3531#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3532
3533static int __init fail_page_alloc_debugfs(void)
3534{
Joe Perches0825a6f2018-06-14 15:27:58 -07003535 umode_t mode = S_IFREG | 0600;
Akinobu Mita933e3122006-12-08 02:39:45 -08003536 struct dentry *dir;
Akinobu Mita933e3122006-12-08 02:39:45 -08003537
Akinobu Mitadd48c082011-08-03 16:21:01 -07003538 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3539 &fail_page_alloc.attr);
Akinobu Mita933e3122006-12-08 02:39:45 -08003540
Greg Kroah-Hartmand9f79792019-03-05 15:46:09 -08003541 debugfs_create_bool("ignore-gfp-wait", mode, dir,
3542 &fail_page_alloc.ignore_gfp_reclaim);
3543 debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3544 &fail_page_alloc.ignore_gfp_highmem);
3545 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
Akinobu Mita933e3122006-12-08 02:39:45 -08003546
Akinobu Mitab2588c42011-07-26 16:09:03 -07003547 return 0;
Akinobu Mita933e3122006-12-08 02:39:45 -08003548}
3549
3550late_initcall(fail_page_alloc_debugfs);
3551
3552#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3553
3554#else /* CONFIG_FAIL_PAGE_ALLOC */
3555
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003556static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08003557{
Gavin Shandeaf3862012-07-31 16:41:51 -07003558 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003559}
3560
3561#endif /* CONFIG_FAIL_PAGE_ALLOC */
3562
Alexei Starovoitov76cd6172020-08-27 15:01:10 -07003563noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003564{
3565 return __should_fail_alloc_page(gfp_mask, order);
3566}
3567ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3568
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003569static inline long __zone_watermark_unusable_free(struct zone *z,
3570 unsigned int order, unsigned int alloc_flags)
3571{
3572 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3573 long unusable_free = (1 << order) - 1;
3574
3575 /*
3576 * If the caller does not have rights to ALLOC_HARDER then subtract
3577 * the high-atomic reserves. This will over-estimate the size of the
3578 * atomic reserve but it avoids a search.
3579 */
3580 if (likely(!alloc_harder))
3581 unusable_free += z->nr_reserved_highatomic;
3582
3583#ifdef CONFIG_CMA
3584 /* If allocation can't use CMA areas don't use free CMA pages */
3585 if (!(alloc_flags & ALLOC_CMA))
3586 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
3587#endif
3588
3589 return unusable_free;
3590}
3591
Linus Torvalds1da177e2005-04-16 15:20:36 -07003592/*
Mel Gorman97a16fc2015-11-06 16:28:40 -08003593 * Return true if free base pages are above 'mark'. For high-order checks it
3594 * will return true of the order-0 watermark is reached and there is at least
3595 * one free page of a suitable size. Checking now avoids taking the zone lock
3596 * to check in the allocation paths if no pages are free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003597 */
Michal Hocko86a294a2016-05-20 16:57:12 -07003598bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003599 int highest_zoneidx, unsigned int alloc_flags,
Michal Hocko86a294a2016-05-20 16:57:12 -07003600 long free_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003601{
Christoph Lameterd23ad422007-02-10 01:43:02 -08003602 long min = mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003603 int o;
Michal Hockocd04ae12017-09-06 16:24:50 -07003604 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003605
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003606 /* free_pages may go negative - that's OK */
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003607 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003608
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003609 if (alloc_flags & ALLOC_HIGH)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003610 min -= min / 2;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003611
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003612 if (unlikely(alloc_harder)) {
Michal Hockocd04ae12017-09-06 16:24:50 -07003613 /*
3614 * OOM victims can try even harder than normal ALLOC_HARDER
3615 * users on the grounds that it's definitely going to be in
3616 * the exit path shortly and free memory. Any allocation it
3617 * makes during the free path will be small and short-lived.
3618 */
3619 if (alloc_flags & ALLOC_OOM)
3620 min -= min / 2;
3621 else
3622 min -= min / 4;
3623 }
3624
Mel Gorman97a16fc2015-11-06 16:28:40 -08003625 /*
3626 * Check watermarks for an order-0 allocation request. If these
3627 * are not met, then a high-order request also cannot go ahead
3628 * even if a suitable page happened to be free.
3629 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003630 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
Mel Gorman88f5acf2011-01-13 15:45:41 -08003631 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003632
Mel Gorman97a16fc2015-11-06 16:28:40 -08003633 /* If this is an order-0 request then the watermark is fine */
3634 if (!order)
3635 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003636
Mel Gorman97a16fc2015-11-06 16:28:40 -08003637 /* For a high-order request, check at least one suitable page is free */
3638 for (o = order; o < MAX_ORDER; o++) {
3639 struct free_area *area = &z->free_area[o];
3640 int mt;
3641
3642 if (!area->nr_free)
3643 continue;
3644
Mel Gorman97a16fc2015-11-06 16:28:40 -08003645 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
Dan Williamsb03641a2019-05-14 15:41:32 -07003646 if (!free_area_empty(area, mt))
Mel Gorman97a16fc2015-11-06 16:28:40 -08003647 return true;
3648 }
3649
3650#ifdef CONFIG_CMA
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003651 if ((alloc_flags & ALLOC_CMA) &&
Dan Williamsb03641a2019-05-14 15:41:32 -07003652 !free_area_empty(area, MIGRATE_CMA)) {
Mel Gorman97a16fc2015-11-06 16:28:40 -08003653 return true;
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003654 }
Mel Gorman97a16fc2015-11-06 16:28:40 -08003655#endif
chenqiwu76089d02020-04-01 21:09:50 -07003656 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))
Vlastimil Babkab050e372017-11-15 17:38:30 -08003657 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003658 }
Mel Gorman97a16fc2015-11-06 16:28:40 -08003659 return false;
Mel Gorman88f5acf2011-01-13 15:45:41 -08003660}
3661
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003662bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003663 int highest_zoneidx, unsigned int alloc_flags)
Mel Gorman88f5acf2011-01-13 15:45:41 -08003664{
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003665 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
Mel Gorman88f5acf2011-01-13 15:45:41 -08003666 zone_page_state(z, NR_FREE_PAGES));
3667}
3668
Mel Gorman48ee5f32016-05-19 17:14:07 -07003669static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003670 unsigned long mark, int highest_zoneidx,
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003671 unsigned int alloc_flags, gfp_t gfp_mask)
Mel Gorman48ee5f32016-05-19 17:14:07 -07003672{
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003673 long free_pages;
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003674
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003675 free_pages = zone_page_state(z, NR_FREE_PAGES);
Mel Gorman48ee5f32016-05-19 17:14:07 -07003676
3677 /*
3678 * Fast check for order-0 only. If this fails then the reserves
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003679 * need to be calculated.
Mel Gorman48ee5f32016-05-19 17:14:07 -07003680 */
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003681 if (!order) {
3682 long fast_free;
3683
3684 fast_free = free_pages;
3685 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags);
3686 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx])
3687 return true;
3688 }
Mel Gorman48ee5f32016-05-19 17:14:07 -07003689
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003690 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3691 free_pages))
3692 return true;
3693 /*
3694 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations
3695 * when checking the min watermark. The min watermark is the
3696 * point where boosting is ignored so that kswapd is woken up
3697 * when below the low watermark.
3698 */
3699 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost
3700 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
3701 mark = z->_watermark[WMARK_MIN];
3702 return __zone_watermark_ok(z, order, mark, highest_zoneidx,
3703 alloc_flags, free_pages);
3704 }
3705
3706 return false;
Mel Gorman48ee5f32016-05-19 17:14:07 -07003707}
3708
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003709bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003710 unsigned long mark, int highest_zoneidx)
Mel Gorman88f5acf2011-01-13 15:45:41 -08003711{
3712 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3713
3714 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3715 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3716
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003717 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,
Mel Gorman88f5acf2011-01-13 15:45:41 -08003718 free_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003719}
3720
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003721#ifdef CONFIG_NUMA
David Rientjes957f8222012-10-08 16:33:24 -07003722static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3723{
Gavin Shane02dc012017-02-24 14:59:33 -08003724 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
Matt Fleminga55c7452019-08-08 20:53:01 +01003725 node_reclaim_distance;
David Rientjes957f8222012-10-08 16:33:24 -07003726}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003727#else /* CONFIG_NUMA */
David Rientjes957f8222012-10-08 16:33:24 -07003728static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3729{
3730 return true;
3731}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003732#endif /* CONFIG_NUMA */
3733
Mel Gorman6bb15452018-12-28 00:35:41 -08003734/*
3735 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3736 * fragmentation is subtle. If the preferred zone was HIGHMEM then
3737 * premature use of a lower zone may cause lowmem pressure problems that
3738 * are worse than fragmentation. If the next zone is ZONE_DMA then it is
3739 * probably too small. It only makes sense to spread allocations to avoid
3740 * fragmentation between the Normal and DMA32 zones.
3741 */
3742static inline unsigned int
Mel Gorman0a79cda2018-12-28 00:35:48 -08003743alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
Mel Gorman6bb15452018-12-28 00:35:41 -08003744{
Mateusz Nosek736838e2020-04-01 21:09:47 -07003745 unsigned int alloc_flags;
Mel Gorman0a79cda2018-12-28 00:35:48 -08003746
Mateusz Nosek736838e2020-04-01 21:09:47 -07003747 /*
3748 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
3749 * to save a branch.
3750 */
3751 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
Mel Gorman0a79cda2018-12-28 00:35:48 -08003752
3753#ifdef CONFIG_ZONE_DMA32
Andrey Ryabinin8139ad02019-04-25 22:23:58 -07003754 if (!zone)
3755 return alloc_flags;
3756
Mel Gorman6bb15452018-12-28 00:35:41 -08003757 if (zone_idx(zone) != ZONE_NORMAL)
Andrey Ryabinin8118b822019-04-25 22:24:01 -07003758 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08003759
3760 /*
3761 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
3762 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
3763 * on UMA that if Normal is populated then so is DMA32.
3764 */
3765 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
3766 if (nr_online_nodes > 1 && !populated_zone(--zone))
Andrey Ryabinin8118b822019-04-25 22:24:01 -07003767 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08003768
Andrey Ryabinin8118b822019-04-25 22:24:01 -07003769 alloc_flags |= ALLOC_NOFRAGMENT;
Mel Gorman0a79cda2018-12-28 00:35:48 -08003770#endif /* CONFIG_ZONE_DMA32 */
3771 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08003772}
Mel Gorman6bb15452018-12-28 00:35:41 -08003773
Joonsoo Kim8510e692020-08-06 23:26:04 -07003774static inline unsigned int current_alloc_flags(gfp_t gfp_mask,
3775 unsigned int alloc_flags)
3776{
3777#ifdef CONFIG_CMA
3778 unsigned int pflags = current->flags;
3779
3780 if (!(pflags & PF_MEMALLOC_NOCMA) &&
3781 gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
3782 alloc_flags |= ALLOC_CMA;
3783
3784#endif
3785 return alloc_flags;
3786}
3787
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003788/*
Paul Jackson0798e512006-12-06 20:31:38 -08003789 * get_page_from_freelist goes through the zonelist trying to allocate
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003790 * a page.
3791 */
3792static struct page *
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003793get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3794 const struct alloc_context *ac)
Martin Hicks753ee722005-06-21 17:14:41 -07003795{
Mel Gorman6bb15452018-12-28 00:35:41 -08003796 struct zoneref *z;
Mel Gorman5117f452009-06-16 15:31:59 -07003797 struct zone *zone;
Mel Gorman3b8c0be2016-07-28 15:46:53 -07003798 struct pglist_data *last_pgdat_dirty_limit = NULL;
Mel Gorman6bb15452018-12-28 00:35:41 -08003799 bool no_fallback;
Mel Gorman3b8c0be2016-07-28 15:46:53 -07003800
Mel Gorman6bb15452018-12-28 00:35:41 -08003801retry:
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003802 /*
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003803 * Scan zonelist, looking for a zone with enough free.
Vladimir Davydov344736f2014-10-20 15:50:30 +04003804 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003805 */
Mel Gorman6bb15452018-12-28 00:35:41 -08003806 no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3807 z = ac->preferred_zoneref;
Mateusz Nosek30d8ec72020-10-13 16:55:57 -07003808 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
3809 ac->nodemask) {
Mel Gormanbe06af02016-05-19 17:13:47 -07003810 struct page *page;
Johannes Weinere085dbc2013-09-11 14:20:46 -07003811 unsigned long mark;
3812
Mel Gorman664eedd2014-06-04 16:10:08 -07003813 if (cpusets_enabled() &&
3814 (alloc_flags & ALLOC_CPUSET) &&
Vlastimil Babka002f2902016-05-19 17:14:30 -07003815 !__cpuset_zone_allowed(zone, gfp_mask))
Mel Gormancd38b112011-07-25 17:12:29 -07003816 continue;
Johannes Weinera756cf52012-01-10 15:07:49 -08003817 /*
3818 * When allocating a page cache page for writing, we
Mel Gorman281e3722016-07-28 15:46:11 -07003819 * want to get it from a node that is within its dirty
3820 * limit, such that no single node holds more than its
Johannes Weinera756cf52012-01-10 15:07:49 -08003821 * proportional share of globally allowed dirty pages.
Mel Gorman281e3722016-07-28 15:46:11 -07003822 * The dirty limits take into account the node's
Johannes Weinera756cf52012-01-10 15:07:49 -08003823 * lowmem reserves and high watermark so that kswapd
3824 * should be able to balance it without having to
3825 * write pages from its LRU list.
3826 *
Johannes Weinera756cf52012-01-10 15:07:49 -08003827 * XXX: For now, allow allocations to potentially
Mel Gorman281e3722016-07-28 15:46:11 -07003828 * exceed the per-node dirty limit in the slowpath
Mel Gormanc9ab0c42015-11-06 16:28:12 -08003829 * (spread_dirty_pages unset) before going into reclaim,
Johannes Weinera756cf52012-01-10 15:07:49 -08003830 * which is important when on a NUMA setup the allowed
Mel Gorman281e3722016-07-28 15:46:11 -07003831 * nodes are together not big enough to reach the
Johannes Weinera756cf52012-01-10 15:07:49 -08003832 * global limit. The proper fix for these situations
Mel Gorman281e3722016-07-28 15:46:11 -07003833 * will require awareness of nodes in the
Johannes Weinera756cf52012-01-10 15:07:49 -08003834 * dirty-throttling and the flusher threads.
3835 */
Mel Gorman3b8c0be2016-07-28 15:46:53 -07003836 if (ac->spread_dirty_pages) {
3837 if (last_pgdat_dirty_limit == zone->zone_pgdat)
3838 continue;
3839
3840 if (!node_dirty_ok(zone->zone_pgdat)) {
3841 last_pgdat_dirty_limit = zone->zone_pgdat;
3842 continue;
3843 }
3844 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003845
Mel Gorman6bb15452018-12-28 00:35:41 -08003846 if (no_fallback && nr_online_nodes > 1 &&
3847 zone != ac->preferred_zoneref->zone) {
3848 int local_nid;
3849
3850 /*
3851 * If moving to a remote node, retry but allow
3852 * fragmenting fallbacks. Locality is more important
3853 * than fragmentation avoidance.
3854 */
3855 local_nid = zone_to_nid(ac->preferred_zoneref->zone);
3856 if (zone_to_nid(zone) != local_nid) {
3857 alloc_flags &= ~ALLOC_NOFRAGMENT;
3858 goto retry;
3859 }
3860 }
3861
Mel Gormana9214442018-12-28 00:35:44 -08003862 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
Mel Gorman48ee5f32016-05-19 17:14:07 -07003863 if (!zone_watermark_fast(zone, order, mark,
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003864 ac->highest_zoneidx, alloc_flags,
3865 gfp_mask)) {
Mel Gormanfa5e0842009-06-16 15:33:22 -07003866 int ret;
3867
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07003868#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3869 /*
3870 * Watermark failed for this zone, but see if we can
3871 * grow this zone if it contains deferred pages.
3872 */
3873 if (static_branch_unlikely(&deferred_pages)) {
3874 if (_deferred_grow_zone(zone, order))
3875 goto try_this_zone;
3876 }
3877#endif
Mel Gorman5dab2912014-06-04 16:10:14 -07003878 /* Checked here to keep the fast path fast */
3879 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3880 if (alloc_flags & ALLOC_NO_WATERMARKS)
3881 goto try_this_zone;
3882
Mel Gormana5f5f912016-07-28 15:46:32 -07003883 if (node_reclaim_mode == 0 ||
Mel Gormanc33d6c02016-05-19 17:14:10 -07003884 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
Mel Gormancd38b112011-07-25 17:12:29 -07003885 continue;
3886
Mel Gormana5f5f912016-07-28 15:46:32 -07003887 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
Mel Gormanfa5e0842009-06-16 15:33:22 -07003888 switch (ret) {
Mel Gormana5f5f912016-07-28 15:46:32 -07003889 case NODE_RECLAIM_NOSCAN:
Mel Gormanfa5e0842009-06-16 15:33:22 -07003890 /* did not scan */
Mel Gormancd38b112011-07-25 17:12:29 -07003891 continue;
Mel Gormana5f5f912016-07-28 15:46:32 -07003892 case NODE_RECLAIM_FULL:
Mel Gormanfa5e0842009-06-16 15:33:22 -07003893 /* scanned but unreclaimable */
Mel Gormancd38b112011-07-25 17:12:29 -07003894 continue;
Mel Gormanfa5e0842009-06-16 15:33:22 -07003895 default:
3896 /* did we reclaim enough */
Mel Gormanfed27192013-04-29 15:07:57 -07003897 if (zone_watermark_ok(zone, order, mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003898 ac->highest_zoneidx, alloc_flags))
Mel Gormanfed27192013-04-29 15:07:57 -07003899 goto try_this_zone;
3900
Mel Gormanfed27192013-04-29 15:07:57 -07003901 continue;
Paul Jackson0798e512006-12-06 20:31:38 -08003902 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003903 }
3904
Mel Gormanfa5e0842009-06-16 15:33:22 -07003905try_this_zone:
Mel Gorman066b2392017-02-24 14:56:26 -08003906 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003907 gfp_mask, alloc_flags, ac->migratetype);
Vlastimil Babka75379192015-02-11 15:25:38 -08003908 if (page) {
Mel Gorman479f8542016-05-19 17:14:35 -07003909 prep_new_page(page, order, gfp_mask, alloc_flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003910
3911 /*
3912 * If this is a high-order atomic allocation then check
3913 * if the pageblock should be reserved for the future
3914 */
3915 if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
3916 reserve_highatomic_pageblock(page, zone, order);
3917
Vlastimil Babka75379192015-02-11 15:25:38 -08003918 return page;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07003919 } else {
3920#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3921 /* Try again if zone has deferred pages */
3922 if (static_branch_unlikely(&deferred_pages)) {
3923 if (_deferred_grow_zone(zone, order))
3924 goto try_this_zone;
3925 }
3926#endif
Vlastimil Babka75379192015-02-11 15:25:38 -08003927 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07003928 }
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003929
Mel Gorman6bb15452018-12-28 00:35:41 -08003930 /*
3931 * It's possible on a UMA machine to get through all zones that are
3932 * fragmented. If avoiding fragmentation, reset and try again.
3933 */
3934 if (no_fallback) {
3935 alloc_flags &= ~ALLOC_NOFRAGMENT;
3936 goto retry;
3937 }
3938
Mel Gorman4ffeaf32014-08-06 16:07:22 -07003939 return NULL;
Martin Hicks753ee722005-06-21 17:14:41 -07003940}
3941
Michal Hocko9af744d2017-02-22 15:46:16 -08003942static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
Dave Hansena238ab52011-05-24 17:12:16 -07003943{
Dave Hansena238ab52011-05-24 17:12:16 -07003944 unsigned int filter = SHOW_MEM_FILTER_NODES;
Dave Hansena238ab52011-05-24 17:12:16 -07003945
3946 /*
3947 * This documents exceptions given to allocations in certain
3948 * contexts that are allowed to allocate outside current's set
3949 * of allowed nodes.
3950 */
3951 if (!(gfp_mask & __GFP_NOMEMALLOC))
Michal Hockocd04ae12017-09-06 16:24:50 -07003952 if (tsk_is_oom_victim(current) ||
Dave Hansena238ab52011-05-24 17:12:16 -07003953 (current->flags & (PF_MEMALLOC | PF_EXITING)))
3954 filter &= ~SHOW_MEM_FILTER_NODES;
Mel Gormand0164ad2015-11-06 16:28:21 -08003955 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
Dave Hansena238ab52011-05-24 17:12:16 -07003956 filter &= ~SHOW_MEM_FILTER_NODES;
3957
Michal Hocko9af744d2017-02-22 15:46:16 -08003958 show_mem(filter, nodemask);
Michal Hockoaa187502017-02-22 15:41:45 -08003959}
3960
Michal Hockoa8e99252017-02-22 15:46:10 -08003961void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
Michal Hockoaa187502017-02-22 15:41:45 -08003962{
3963 struct va_format vaf;
3964 va_list args;
Johannes Weiner1be334e2019-11-05 21:16:51 -08003965 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
Michal Hockoaa187502017-02-22 15:41:45 -08003966
Baoquan He6c6f86b2022-01-14 14:07:44 -08003967 if ((gfp_mask & __GFP_NOWARN) ||
3968 !__ratelimit(&nopage_rs) ||
3969 ((gfp_mask & __GFP_DMA) && !has_managed_dma()))
Michal Hockoaa187502017-02-22 15:41:45 -08003970 return;
3971
Michal Hocko7877cdc2016-10-07 17:01:55 -07003972 va_start(args, fmt);
3973 vaf.fmt = fmt;
3974 vaf.va = &args;
yuzhoujianef8444e2018-12-28 00:36:07 -08003975 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
Michal Hocko0205f752017-11-15 17:39:14 -08003976 current->comm, &vaf, gfp_mask, &gfp_mask,
3977 nodemask_pr_args(nodemask));
Michal Hocko7877cdc2016-10-07 17:01:55 -07003978 va_end(args);
Joe Perches3ee9a4f2011-10-31 17:08:35 -07003979
Michal Hockoa8e99252017-02-22 15:46:10 -08003980 cpuset_print_current_mems_allowed();
yuzhoujianef8444e2018-12-28 00:36:07 -08003981 pr_cont("\n");
Dave Hansena238ab52011-05-24 17:12:16 -07003982 dump_stack();
David Rientjes685dbf62017-02-22 15:46:28 -08003983 warn_alloc_show_mem(gfp_mask, nodemask);
Dave Hansena238ab52011-05-24 17:12:16 -07003984}
3985
Mel Gorman11e33f62009-06-16 15:31:57 -07003986static inline struct page *
Michal Hocko6c18ba72017-02-22 15:46:25 -08003987__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
3988 unsigned int alloc_flags,
3989 const struct alloc_context *ac)
3990{
3991 struct page *page;
3992
3993 page = get_page_from_freelist(gfp_mask, order,
3994 alloc_flags|ALLOC_CPUSET, ac);
3995 /*
3996 * fallback to ignore cpuset restriction if our nodes
3997 * are depleted
3998 */
3999 if (!page)
4000 page = get_page_from_freelist(gfp_mask, order,
4001 alloc_flags, ac);
4002
4003 return page;
4004}
4005
4006static inline struct page *
Mel Gorman11e33f62009-06-16 15:31:57 -07004007__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004008 const struct alloc_context *ac, unsigned long *did_some_progress)
Mel Gorman11e33f62009-06-16 15:31:57 -07004009{
David Rientjes6e0fc462015-09-08 15:00:36 -07004010 struct oom_control oc = {
4011 .zonelist = ac->zonelist,
4012 .nodemask = ac->nodemask,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07004013 .memcg = NULL,
David Rientjes6e0fc462015-09-08 15:00:36 -07004014 .gfp_mask = gfp_mask,
4015 .order = order,
David Rientjes6e0fc462015-09-08 15:00:36 -07004016 };
Linus Torvalds1da177e2005-04-16 15:20:36 -07004017 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004018
Johannes Weiner9879de72015-01-26 12:58:32 -08004019 *did_some_progress = 0;
4020
Johannes Weiner9879de72015-01-26 12:58:32 -08004021 /*
Johannes Weinerdc564012015-06-24 16:57:19 -07004022 * Acquire the oom lock. If that fails, somebody else is
4023 * making progress for us.
Johannes Weiner9879de72015-01-26 12:58:32 -08004024 */
Johannes Weinerdc564012015-06-24 16:57:19 -07004025 if (!mutex_trylock(&oom_lock)) {
Johannes Weiner9879de72015-01-26 12:58:32 -08004026 *did_some_progress = 1;
Mel Gorman11e33f62009-06-16 15:31:57 -07004027 schedule_timeout_uninterruptible(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004028 return NULL;
4029 }
Jens Axboe6b1de912005-11-17 21:35:02 +01004030
Mel Gorman11e33f62009-06-16 15:31:57 -07004031 /*
4032 * Go through the zonelist yet one more time, keep very high watermark
4033 * here, this is only to catch a parallel oom killing, we must fail if
Tetsuo Handae746bf72017-08-31 16:15:20 -07004034 * we're still under heavy pressure. But make sure that this reclaim
4035 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
4036 * allocation which will never fail due to oom_lock already held.
Mel Gorman11e33f62009-06-16 15:31:57 -07004037 */
Tetsuo Handae746bf72017-08-31 16:15:20 -07004038 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
4039 ~__GFP_DIRECT_RECLAIM, order,
4040 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004041 if (page)
Mel Gorman11e33f62009-06-16 15:31:57 -07004042 goto out;
4043
Michal Hocko06ad2762017-02-22 15:46:22 -08004044 /* Coredumps can quickly deplete all memory reserves */
4045 if (current->flags & PF_DUMPCORE)
4046 goto out;
4047 /* The OOM killer will not help higher order allocs */
4048 if (order > PAGE_ALLOC_COSTLY_ORDER)
4049 goto out;
Michal Hockodcda9b02017-07-12 14:36:45 -07004050 /*
4051 * We have already exhausted all our reclaim opportunities without any
4052 * success so it is time to admit defeat. We will skip the OOM killer
4053 * because it is very likely that the caller has a more reasonable
4054 * fallback than shooting a random task.
Mateusz Nosekcfb4a542020-10-13 16:55:45 -07004055 *
4056 * The OOM killer may not free memory on a specific node.
Michal Hockodcda9b02017-07-12 14:36:45 -07004057 */
Mateusz Nosekcfb4a542020-10-13 16:55:45 -07004058 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
Michal Hockodcda9b02017-07-12 14:36:45 -07004059 goto out;
Michal Hocko06ad2762017-02-22 15:46:22 -08004060 /* The OOM killer does not needlessly kill tasks for lowmem */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004061 if (ac->highest_zoneidx < ZONE_NORMAL)
Michal Hocko06ad2762017-02-22 15:46:22 -08004062 goto out;
4063 if (pm_suspended_storage())
4064 goto out;
4065 /*
4066 * XXX: GFP_NOFS allocations should rather fail than rely on
4067 * other request to make a forward progress.
4068 * We are in an unfortunate situation where out_of_memory cannot
4069 * do much for this context but let's try it to at least get
4070 * access to memory reserved if the current task is killed (see
4071 * out_of_memory). Once filesystems are ready to handle allocation
4072 * failures more gracefully we should just bail out here.
4073 */
Michal Hocko3da88fb32016-05-19 17:13:09 -07004074
Shile Zhang3c2c6482018-01-31 16:20:07 -08004075 /* Exhausted what can be done so it's blame time */
Michal Hocko5020e282016-01-14 15:20:36 -08004076 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
Michal Hockoc32b3cb2015-02-11 15:26:24 -08004077 *did_some_progress = 1;
Michal Hocko5020e282016-01-14 15:20:36 -08004078
Michal Hocko6c18ba72017-02-22 15:46:25 -08004079 /*
4080 * Help non-failing allocations by giving them access to memory
4081 * reserves
4082 */
4083 if (gfp_mask & __GFP_NOFAIL)
4084 page = __alloc_pages_cpuset_fallback(gfp_mask, order,
Michal Hocko5020e282016-01-14 15:20:36 -08004085 ALLOC_NO_WATERMARKS, ac);
Michal Hocko5020e282016-01-14 15:20:36 -08004086 }
Mel Gorman11e33f62009-06-16 15:31:57 -07004087out:
Johannes Weinerdc564012015-06-24 16:57:19 -07004088 mutex_unlock(&oom_lock);
Mel Gorman11e33f62009-06-16 15:31:57 -07004089 return page;
4090}
4091
Michal Hocko33c2d212016-05-20 16:57:06 -07004092/*
4093 * Maximum number of compaction retries wit a progress before OOM
4094 * killer is consider as the only way to move forward.
4095 */
4096#define MAX_COMPACT_RETRIES 16
4097
Mel Gorman56de7262010-05-24 14:32:30 -07004098#ifdef CONFIG_COMPACTION
4099/* Try memory compaction for high-order allocations before reclaim */
4100static struct page *
4101__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004102 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004103 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07004104{
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004105 struct page *page = NULL;
Johannes Weinereb414682018-10-26 15:06:27 -07004106 unsigned long pflags;
Vlastimil Babka499118e2017-05-08 15:59:50 -07004107 unsigned int noreclaim_flag;
Vlastimil Babka53853e22014-10-09 15:27:02 -07004108
Mel Gorman66199712012-01-12 17:19:41 -08004109 if (!order)
Mel Gorman56de7262010-05-24 14:32:30 -07004110 return NULL;
4111
Johannes Weinereb414682018-10-26 15:06:27 -07004112 psi_memstall_enter(&pflags);
Vlastimil Babka499118e2017-05-08 15:59:50 -07004113 noreclaim_flag = memalloc_noreclaim_save();
Johannes Weinereb414682018-10-26 15:06:27 -07004114
Michal Hockoc5d01d02016-05-20 16:56:53 -07004115 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004116 prio, &page);
Johannes Weinereb414682018-10-26 15:06:27 -07004117
Vlastimil Babka499118e2017-05-08 15:59:50 -07004118 memalloc_noreclaim_restore(noreclaim_flag);
Johannes Weinereb414682018-10-26 15:06:27 -07004119 psi_memstall_leave(&pflags);
Mel Gorman56de7262010-05-24 14:32:30 -07004120
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004121 /*
4122 * At least in one zone compaction wasn't deferred or skipped, so let's
4123 * count a compaction stall
4124 */
4125 count_vm_event(COMPACTSTALL);
4126
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004127 /* Prep a captured page if available */
4128 if (page)
4129 prep_new_page(page, order, gfp_mask, alloc_flags);
4130
4131 /* Try get a page from the freelist if available */
4132 if (!page)
4133 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004134
4135 if (page) {
4136 struct zone *zone = page_zone(page);
4137
4138 zone->compact_blockskip_flush = false;
4139 compaction_defer_reset(zone, order, true);
4140 count_vm_event(COMPACTSUCCESS);
4141 return page;
4142 }
4143
4144 /*
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004145 * It's bad if compaction run occurs and fails. The most likely reason
4146 * is that pages exist, but not enough to satisfy watermarks.
4147 */
4148 count_vm_event(COMPACTFAIL);
4149
4150 cond_resched();
4151
Mel Gorman56de7262010-05-24 14:32:30 -07004152 return NULL;
4153}
Michal Hocko33c2d212016-05-20 16:57:06 -07004154
Vlastimil Babka32508452016-10-07 17:00:28 -07004155static inline bool
4156should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
4157 enum compact_result compact_result,
4158 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004159 int *compaction_retries)
Vlastimil Babka32508452016-10-07 17:00:28 -07004160{
4161 int max_retries = MAX_COMPACT_RETRIES;
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004162 int min_priority;
Michal Hocko65190cf2017-02-22 15:42:03 -08004163 bool ret = false;
4164 int retries = *compaction_retries;
4165 enum compact_priority priority = *compact_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07004166
4167 if (!order)
4168 return false;
4169
Vlastimil Babkad9436492016-10-07 17:00:31 -07004170 if (compaction_made_progress(compact_result))
4171 (*compaction_retries)++;
4172
Vlastimil Babka32508452016-10-07 17:00:28 -07004173 /*
4174 * compaction considers all the zone as desperately out of memory
4175 * so it doesn't really make much sense to retry except when the
4176 * failure could be caused by insufficient priority
4177 */
Vlastimil Babkad9436492016-10-07 17:00:31 -07004178 if (compaction_failed(compact_result))
4179 goto check_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07004180
4181 /*
Vlastimil Babka494330852019-09-23 15:37:32 -07004182 * compaction was skipped because there are not enough order-0 pages
4183 * to work with, so we retry only if it looks like reclaim can help.
Vlastimil Babka32508452016-10-07 17:00:28 -07004184 */
Vlastimil Babka494330852019-09-23 15:37:32 -07004185 if (compaction_needs_reclaim(compact_result)) {
Michal Hocko65190cf2017-02-22 15:42:03 -08004186 ret = compaction_zonelist_suitable(ac, order, alloc_flags);
4187 goto out;
4188 }
Vlastimil Babka32508452016-10-07 17:00:28 -07004189
4190 /*
Vlastimil Babka494330852019-09-23 15:37:32 -07004191 * make sure the compaction wasn't deferred or didn't bail out early
4192 * due to locks contention before we declare that we should give up.
4193 * But the next retry should use a higher priority if allowed, so
4194 * we don't just keep bailing out endlessly.
4195 */
4196 if (compaction_withdrawn(compact_result)) {
4197 goto check_priority;
4198 }
4199
4200 /*
Michal Hockodcda9b02017-07-12 14:36:45 -07004201 * !costly requests are much more important than __GFP_RETRY_MAYFAIL
Vlastimil Babka32508452016-10-07 17:00:28 -07004202 * costly ones because they are de facto nofail and invoke OOM
4203 * killer to move on while costly can fail and users are ready
4204 * to cope with that. 1/4 retries is rather arbitrary but we
4205 * would need much more detailed feedback from compaction to
4206 * make a better decision.
4207 */
4208 if (order > PAGE_ALLOC_COSTLY_ORDER)
4209 max_retries /= 4;
Michal Hocko65190cf2017-02-22 15:42:03 -08004210 if (*compaction_retries <= max_retries) {
4211 ret = true;
4212 goto out;
4213 }
Vlastimil Babka32508452016-10-07 17:00:28 -07004214
Vlastimil Babkad9436492016-10-07 17:00:31 -07004215 /*
4216 * Make sure there are attempts at the highest priority if we exhausted
4217 * all retries or failed at the lower priorities.
4218 */
4219check_priority:
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004220 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
4221 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
Michal Hocko65190cf2017-02-22 15:42:03 -08004222
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004223 if (*compact_priority > min_priority) {
Vlastimil Babkad9436492016-10-07 17:00:31 -07004224 (*compact_priority)--;
4225 *compaction_retries = 0;
Michal Hocko65190cf2017-02-22 15:42:03 -08004226 ret = true;
Vlastimil Babkad9436492016-10-07 17:00:31 -07004227 }
Michal Hocko65190cf2017-02-22 15:42:03 -08004228out:
4229 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
4230 return ret;
Vlastimil Babka32508452016-10-07 17:00:28 -07004231}
Mel Gorman56de7262010-05-24 14:32:30 -07004232#else
4233static inline struct page *
4234__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004235 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004236 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07004237{
Michal Hocko33c2d212016-05-20 16:57:06 -07004238 *compact_result = COMPACT_SKIPPED;
Mel Gorman56de7262010-05-24 14:32:30 -07004239 return NULL;
4240}
Michal Hocko33c2d212016-05-20 16:57:06 -07004241
4242static inline bool
Michal Hocko86a294a2016-05-20 16:57:12 -07004243should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
4244 enum compact_result compact_result,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004245 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004246 int *compaction_retries)
Michal Hocko33c2d212016-05-20 16:57:06 -07004247{
Michal Hocko31e49bf2016-05-20 16:57:15 -07004248 struct zone *zone;
4249 struct zoneref *z;
4250
4251 if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
4252 return false;
4253
4254 /*
4255 * There are setups with compaction disabled which would prefer to loop
4256 * inside the allocator rather than hit the oom killer prematurely.
4257 * Let's give them a good hope and keep retrying while the order-0
4258 * watermarks are OK.
4259 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004260 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4261 ac->highest_zoneidx, ac->nodemask) {
Michal Hocko31e49bf2016-05-20 16:57:15 -07004262 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004263 ac->highest_zoneidx, alloc_flags))
Michal Hocko31e49bf2016-05-20 16:57:15 -07004264 return true;
4265 }
Michal Hocko33c2d212016-05-20 16:57:06 -07004266 return false;
4267}
Vlastimil Babka32508452016-10-07 17:00:28 -07004268#endif /* CONFIG_COMPACTION */
Mel Gorman56de7262010-05-24 14:32:30 -07004269
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004270#ifdef CONFIG_LOCKDEP
Omar Sandoval93781322018-06-07 17:07:02 -07004271static struct lockdep_map __fs_reclaim_map =
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004272 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
4273
4274static bool __need_fs_reclaim(gfp_t gfp_mask)
4275{
4276 gfp_mask = current_gfp_context(gfp_mask);
4277
4278 /* no reclaim without waiting on it */
4279 if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
4280 return false;
4281
4282 /* this guy won't enter reclaim */
Tetsuo Handa2e517d682018-03-22 16:17:10 -07004283 if (current->flags & PF_MEMALLOC)
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004284 return false;
4285
4286 /* We're only interested __GFP_FS allocations for now */
4287 if (!(gfp_mask & __GFP_FS))
4288 return false;
4289
4290 if (gfp_mask & __GFP_NOLOCKDEP)
4291 return false;
4292
4293 return true;
4294}
4295
Omar Sandoval93781322018-06-07 17:07:02 -07004296void __fs_reclaim_acquire(void)
4297{
4298 lock_map_acquire(&__fs_reclaim_map);
4299}
4300
4301void __fs_reclaim_release(void)
4302{
4303 lock_map_release(&__fs_reclaim_map);
4304}
4305
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004306void fs_reclaim_acquire(gfp_t gfp_mask)
4307{
4308 if (__need_fs_reclaim(gfp_mask))
Omar Sandoval93781322018-06-07 17:07:02 -07004309 __fs_reclaim_acquire();
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004310}
4311EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4312
4313void fs_reclaim_release(gfp_t gfp_mask)
4314{
4315 if (__need_fs_reclaim(gfp_mask))
Omar Sandoval93781322018-06-07 17:07:02 -07004316 __fs_reclaim_release();
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004317}
4318EXPORT_SYMBOL_GPL(fs_reclaim_release);
4319#endif
4320
Marek Szyprowskibba90712012-01-25 12:09:52 +01004321/* Perform direct synchronous page reclaim */
Yanfei Xu2187e172020-10-13 16:55:54 -07004322static unsigned long
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004323__perform_reclaim(gfp_t gfp_mask, unsigned int order,
4324 const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004325{
Vlastimil Babka499118e2017-05-08 15:59:50 -07004326 unsigned int noreclaim_flag;
Yanfei Xu2187e172020-10-13 16:55:54 -07004327 unsigned long pflags, progress;
Mel Gorman11e33f62009-06-16 15:31:57 -07004328
4329 cond_resched();
4330
4331 /* We now go into synchronous reclaim */
4332 cpuset_memory_pressure_bump();
Johannes Weinereb414682018-10-26 15:06:27 -07004333 psi_memstall_enter(&pflags);
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004334 fs_reclaim_acquire(gfp_mask);
Omar Sandoval93781322018-06-07 17:07:02 -07004335 noreclaim_flag = memalloc_noreclaim_save();
Mel Gorman11e33f62009-06-16 15:31:57 -07004336
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004337 progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4338 ac->nodemask);
Mel Gorman11e33f62009-06-16 15:31:57 -07004339
Vlastimil Babka499118e2017-05-08 15:59:50 -07004340 memalloc_noreclaim_restore(noreclaim_flag);
Omar Sandoval93781322018-06-07 17:07:02 -07004341 fs_reclaim_release(gfp_mask);
Johannes Weinereb414682018-10-26 15:06:27 -07004342 psi_memstall_leave(&pflags);
Mel Gorman11e33f62009-06-16 15:31:57 -07004343
4344 cond_resched();
4345
Marek Szyprowskibba90712012-01-25 12:09:52 +01004346 return progress;
4347}
4348
4349/* The really slow allocator path where we enter direct reclaim */
4350static inline struct page *
4351__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004352 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004353 unsigned long *did_some_progress)
Marek Szyprowskibba90712012-01-25 12:09:52 +01004354{
4355 struct page *page = NULL;
4356 bool drained = false;
4357
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004358 *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004359 if (unlikely(!(*did_some_progress)))
4360 return NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07004361
Mel Gorman9ee493c2010-09-09 16:38:18 -07004362retry:
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004363 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004364
4365 /*
4366 * If an allocation failed after direct reclaim, it could be because
Mel Gorman0aaa29a2015-11-06 16:28:37 -08004367 * pages are pinned on the per-cpu lists or in high alloc reserves.
Randy Dunlap047b9962020-08-11 18:33:14 -07004368 * Shrink them and try again
Mel Gorman9ee493c2010-09-09 16:38:18 -07004369 */
4370 if (!page && !drained) {
Minchan Kim29fac032016-12-12 16:42:14 -08004371 unreserve_highatomic_pageblock(ac, false);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08004372 drain_all_pages(NULL);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004373 drained = true;
4374 goto retry;
4375 }
4376
Mel Gorman11e33f62009-06-16 15:31:57 -07004377 return page;
4378}
4379
David Rientjes5ecd9d42018-04-05 16:25:16 -07004380static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4381 const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004382{
4383 struct zoneref *z;
4384 struct zone *zone;
Mel Gormane1a55632016-07-28 15:46:26 -07004385 pg_data_t *last_pgdat = NULL;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004386 enum zone_type highest_zoneidx = ac->highest_zoneidx;
Mel Gorman11e33f62009-06-16 15:31:57 -07004387
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004388 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
David Rientjes5ecd9d42018-04-05 16:25:16 -07004389 ac->nodemask) {
Mel Gormane1a55632016-07-28 15:46:26 -07004390 if (last_pgdat != zone->zone_pgdat)
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004391 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);
Mel Gormane1a55632016-07-28 15:46:26 -07004392 last_pgdat = zone->zone_pgdat;
4393 }
Mel Gorman11e33f62009-06-16 15:31:57 -07004394}
4395
Mel Gormanc6038442016-05-19 17:13:38 -07004396static inline unsigned int
Peter Zijlstra341ce062009-06-16 15:32:02 -07004397gfp_to_alloc_flags(gfp_t gfp_mask)
4398{
Mel Gormanc6038442016-05-19 17:13:38 -07004399 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
Peter Zijlstra341ce062009-06-16 15:32:02 -07004400
Mateusz Nosek736838e2020-04-01 21:09:47 -07004401 /*
4402 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH
4403 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4404 * to save two branches.
4405 */
Namhyung Kime6223a32010-10-26 14:21:59 -07004406 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
Mateusz Nosek736838e2020-04-01 21:09:47 -07004407 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
Mel Gormana56f57f2009-06-16 15:32:02 -07004408
Peter Zijlstra341ce062009-06-16 15:32:02 -07004409 /*
4410 * The caller may dip into page reserves a bit more if the caller
4411 * cannot run direct reclaim, or if the caller has realtime scheduling
4412 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
Mel Gormand0164ad2015-11-06 16:28:21 -08004413 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
Peter Zijlstra341ce062009-06-16 15:32:02 -07004414 */
Mateusz Nosek736838e2020-04-01 21:09:47 -07004415 alloc_flags |= (__force int)
4416 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
Peter Zijlstra341ce062009-06-16 15:32:02 -07004417
Mel Gormand0164ad2015-11-06 16:28:21 -08004418 if (gfp_mask & __GFP_ATOMIC) {
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004419 /*
David Rientjesb104a352014-07-30 16:08:24 -07004420 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
4421 * if it can't schedule.
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004422 */
David Rientjesb104a352014-07-30 16:08:24 -07004423 if (!(gfp_mask & __GFP_NOMEMALLOC))
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004424 alloc_flags |= ALLOC_HARDER;
Peter Zijlstra341ce062009-06-16 15:32:02 -07004425 /*
David Rientjesb104a352014-07-30 16:08:24 -07004426 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
Vladimir Davydov344736f2014-10-20 15:50:30 +04004427 * comment for __cpuset_node_allowed().
Peter Zijlstra341ce062009-06-16 15:32:02 -07004428 */
4429 alloc_flags &= ~ALLOC_CPUSET;
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08004430 } else if (unlikely(rt_task(current)) && !in_interrupt())
Peter Zijlstra341ce062009-06-16 15:32:02 -07004431 alloc_flags |= ALLOC_HARDER;
4432
Joonsoo Kim8510e692020-08-06 23:26:04 -07004433 alloc_flags = current_alloc_flags(gfp_mask, alloc_flags);
4434
Peter Zijlstra341ce062009-06-16 15:32:02 -07004435 return alloc_flags;
4436}
4437
Michal Hockocd04ae12017-09-06 16:24:50 -07004438static bool oom_reserves_allowed(struct task_struct *tsk)
Mel Gorman072bb0a2012-07-31 16:43:58 -07004439{
Michal Hockocd04ae12017-09-06 16:24:50 -07004440 if (!tsk_is_oom_victim(tsk))
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004441 return false;
4442
Michal Hockocd04ae12017-09-06 16:24:50 -07004443 /*
4444 * !MMU doesn't have oom reaper so give access to memory reserves
4445 * only to the thread with TIF_MEMDIE set
4446 */
4447 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4448 return false;
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004449
Michal Hockocd04ae12017-09-06 16:24:50 -07004450 return true;
4451}
4452
4453/*
4454 * Distinguish requests which really need access to full memory
4455 * reserves from oom victims which can live with a portion of it
4456 */
4457static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4458{
4459 if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4460 return 0;
4461 if (gfp_mask & __GFP_MEMALLOC)
4462 return ALLOC_NO_WATERMARKS;
4463 if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4464 return ALLOC_NO_WATERMARKS;
4465 if (!in_interrupt()) {
4466 if (current->flags & PF_MEMALLOC)
4467 return ALLOC_NO_WATERMARKS;
4468 else if (oom_reserves_allowed(current))
4469 return ALLOC_OOM;
4470 }
4471
4472 return 0;
4473}
4474
4475bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4476{
4477 return !!__gfp_pfmemalloc_flags(gfp_mask);
Mel Gorman072bb0a2012-07-31 16:43:58 -07004478}
4479
Michal Hocko0a0337e2016-05-20 16:57:00 -07004480/*
Michal Hocko0a0337e2016-05-20 16:57:00 -07004481 * Checks whether it makes sense to retry the reclaim to make a forward progress
4482 * for the given allocation request.
Johannes Weiner491d79a2017-05-03 14:52:16 -07004483 *
4484 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
4485 * without success, or when we couldn't even meet the watermark if we
4486 * reclaimed all remaining pages on the LRU lists.
Michal Hocko0a0337e2016-05-20 16:57:00 -07004487 *
4488 * Returns true if a retry is viable or false to enter the oom path.
4489 */
4490static inline bool
4491should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4492 struct alloc_context *ac, int alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07004493 bool did_some_progress, int *no_progress_loops)
Michal Hocko0a0337e2016-05-20 16:57:00 -07004494{
4495 struct zone *zone;
4496 struct zoneref *z;
Michal Hocko15f570b2018-10-26 15:03:31 -07004497 bool ret = false;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004498
4499 /*
Vlastimil Babka423b4522016-10-07 17:00:40 -07004500 * Costly allocations might have made a progress but this doesn't mean
4501 * their order will become available due to high fragmentation so
4502 * always increment the no progress counter for them
4503 */
4504 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4505 *no_progress_loops = 0;
4506 else
4507 (*no_progress_loops)++;
4508
4509 /*
Michal Hocko0a0337e2016-05-20 16:57:00 -07004510 * Make sure we converge to OOM if we cannot make any progress
4511 * several times in the row.
4512 */
Minchan Kim04c87162016-12-12 16:42:11 -08004513 if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4514 /* Before OOM, exhaust highatomic_reserve */
Minchan Kim29fac032016-12-12 16:42:14 -08004515 return unreserve_highatomic_pageblock(ac, true);
Minchan Kim04c87162016-12-12 16:42:11 -08004516 }
Michal Hocko0a0337e2016-05-20 16:57:00 -07004517
Michal Hocko0a0337e2016-05-20 16:57:00 -07004518 /*
Mel Gormanbca67592016-07-28 15:47:05 -07004519 * Keep reclaiming pages while there is a chance this will lead
4520 * somewhere. If none of the target zones can satisfy our allocation
4521 * request even if all reclaimable pages are considered then we are
4522 * screwed and have to go OOM.
Michal Hocko0a0337e2016-05-20 16:57:00 -07004523 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004524 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4525 ac->highest_zoneidx, ac->nodemask) {
Michal Hocko0a0337e2016-05-20 16:57:00 -07004526 unsigned long available;
Michal Hockoede37712016-05-20 16:57:03 -07004527 unsigned long reclaimable;
Michal Hockod379f012017-02-22 15:42:00 -08004528 unsigned long min_wmark = min_wmark_pages(zone);
4529 bool wmark;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004530
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004531 available = reclaimable = zone_reclaimable_pages(zone);
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004532 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
Michal Hocko0a0337e2016-05-20 16:57:00 -07004533
4534 /*
Johannes Weiner491d79a2017-05-03 14:52:16 -07004535 * Would the allocation succeed if we reclaimed all
4536 * reclaimable pages?
Michal Hocko0a0337e2016-05-20 16:57:00 -07004537 */
Michal Hockod379f012017-02-22 15:42:00 -08004538 wmark = __zone_watermark_ok(zone, order, min_wmark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004539 ac->highest_zoneidx, alloc_flags, available);
Michal Hockod379f012017-02-22 15:42:00 -08004540 trace_reclaim_retry_zone(z, order, reclaimable,
4541 available, min_wmark, *no_progress_loops, wmark);
4542 if (wmark) {
Michal Hockoede37712016-05-20 16:57:03 -07004543 /*
4544 * If we didn't make any progress and have a lot of
4545 * dirty + writeback pages then we should wait for
4546 * an IO to complete to slow down the reclaim and
4547 * prevent from pre mature OOM
4548 */
4549 if (!did_some_progress) {
Mel Gorman11fb9982016-07-28 15:46:20 -07004550 unsigned long write_pending;
Michal Hockoede37712016-05-20 16:57:03 -07004551
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004552 write_pending = zone_page_state_snapshot(zone,
4553 NR_ZONE_WRITE_PENDING);
Michal Hockoede37712016-05-20 16:57:03 -07004554
Mel Gorman11fb9982016-07-28 15:46:20 -07004555 if (2 * write_pending > reclaimable) {
Michal Hockoede37712016-05-20 16:57:03 -07004556 congestion_wait(BLK_RW_ASYNC, HZ/10);
4557 return true;
4558 }
4559 }
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004560
Michal Hocko15f570b2018-10-26 15:03:31 -07004561 ret = true;
4562 goto out;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004563 }
4564 }
4565
Michal Hocko15f570b2018-10-26 15:03:31 -07004566out:
4567 /*
4568 * Memory allocation/reclaim might be called from a WQ context and the
4569 * current implementation of the WQ concurrency control doesn't
4570 * recognize that a particular WQ is congested if the worker thread is
4571 * looping without ever sleeping. Therefore we have to do a short sleep
4572 * here rather than calling cond_resched().
4573 */
4574 if (current->flags & PF_WQ_WORKER)
4575 schedule_timeout_uninterruptible(1);
4576 else
4577 cond_resched();
4578 return ret;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004579}
4580
Vlastimil Babka902b6282017-07-06 15:39:56 -07004581static inline bool
4582check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4583{
4584 /*
4585 * It's possible that cpuset's mems_allowed and the nodemask from
4586 * mempolicy don't intersect. This should be normally dealt with by
4587 * policy_nodemask(), but it's possible to race with cpuset update in
4588 * such a way the check therein was true, and then it became false
4589 * before we got our cpuset_mems_cookie here.
4590 * This assumes that for all allocations, ac->nodemask can come only
4591 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
4592 * when it does not intersect with the cpuset restrictions) or the
4593 * caller can deal with a violated nodemask.
4594 */
4595 if (cpusets_enabled() && ac->nodemask &&
4596 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4597 ac->nodemask = NULL;
4598 return true;
4599 }
4600
4601 /*
4602 * When updating a task's mems_allowed or mempolicy nodemask, it is
4603 * possible to race with parallel threads in such a way that our
4604 * allocation can fail while the mask is being updated. If we are about
4605 * to fail, check if the cpuset changed during allocation and if so,
4606 * retry.
4607 */
4608 if (read_mems_allowed_retry(cpuset_mems_cookie))
4609 return true;
4610
4611 return false;
4612}
4613
Mel Gorman11e33f62009-06-16 15:31:57 -07004614static inline struct page *
4615__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004616 struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004617{
Mel Gormand0164ad2015-11-06 16:28:21 -08004618 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
Vlastimil Babka282722b2017-05-08 15:54:49 -07004619 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
Mel Gorman11e33f62009-06-16 15:31:57 -07004620 struct page *page = NULL;
Mel Gormanc6038442016-05-19 17:13:38 -07004621 unsigned int alloc_flags;
Mel Gorman11e33f62009-06-16 15:31:57 -07004622 unsigned long did_some_progress;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004623 enum compact_priority compact_priority;
Michal Hockoc5d01d02016-05-20 16:56:53 -07004624 enum compact_result compact_result;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004625 int compaction_retries;
4626 int no_progress_loops;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004627 unsigned int cpuset_mems_cookie;
Michal Hockocd04ae12017-09-06 16:24:50 -07004628 int reserve_flags;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004629
Christoph Lameter952f3b52006-12-06 20:33:26 -08004630 /*
Mel Gormand0164ad2015-11-06 16:28:21 -08004631 * We also sanity check to catch abuse of atomic reserves being used by
4632 * callers that are not in atomic context.
4633 */
4634 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4635 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4636 gfp_mask &= ~__GFP_ATOMIC;
4637
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004638retry_cpuset:
4639 compaction_retries = 0;
4640 no_progress_loops = 0;
4641 compact_priority = DEF_COMPACT_PRIORITY;
4642 cpuset_mems_cookie = read_mems_allowed_begin();
Michal Hocko9a67f642017-02-22 15:46:19 -08004643
4644 /*
4645 * The fast path uses conservative alloc_flags to succeed only until
4646 * kswapd needs to be woken up, and to avoid the cost of setting up
4647 * alloc_flags precisely. So we do that now.
4648 */
4649 alloc_flags = gfp_to_alloc_flags(gfp_mask);
4650
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004651 /*
4652 * We need to recalculate the starting point for the zonelist iterator
4653 * because we might have used different nodemask in the fast path, or
4654 * there was a cpuset modification and we are retrying - otherwise we
4655 * could end up iterating over non-eligible zones endlessly.
4656 */
4657 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004658 ac->highest_zoneidx, ac->nodemask);
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004659 if (!ac->preferred_zoneref->zone)
4660 goto nopage;
4661
Mel Gorman0a79cda2018-12-28 00:35:48 -08004662 if (alloc_flags & ALLOC_KSWAPD)
David Rientjes5ecd9d42018-04-05 16:25:16 -07004663 wake_all_kswapds(order, gfp_mask, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004664
Paul Jackson9bf22292005-09-06 15:18:12 -07004665 /*
Vlastimil Babka23771232016-07-28 15:49:16 -07004666 * The adjusted alloc_flags might result in immediate success, so try
4667 * that first
4668 */
4669 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4670 if (page)
4671 goto got_pg;
4672
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004673 /*
4674 * For costly allocations, try direct compaction first, as it's likely
Vlastimil Babka282722b2017-05-08 15:54:49 -07004675 * that we have enough base pages and don't need to reclaim. For non-
4676 * movable high-order allocations, do that as well, as compaction will
4677 * try prevent permanent fragmentation by migrating from blocks of the
4678 * same migratetype.
4679 * Don't try this for allocations that are allowed to ignore
4680 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004681 */
Vlastimil Babka282722b2017-05-08 15:54:49 -07004682 if (can_direct_reclaim &&
4683 (costly_order ||
4684 (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4685 && !gfp_pfmemalloc_allowed(gfp_mask)) {
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004686 page = __alloc_pages_direct_compact(gfp_mask, order,
4687 alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004688 INIT_COMPACT_PRIORITY,
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004689 &compact_result);
4690 if (page)
4691 goto got_pg;
4692
Vlastimil Babkacc638f32020-01-13 16:29:04 -08004693 /*
4694 * Checks for costly allocations with __GFP_NORETRY, which
4695 * includes some THP page fault allocations
4696 */
4697 if (costly_order && (gfp_mask & __GFP_NORETRY)) {
David Rientjesb39d0ee2019-09-04 12:54:22 -07004698 /*
4699 * If allocating entire pageblock(s) and compaction
4700 * failed because all zones are below low watermarks
4701 * or is prohibited because it recently failed at this
David Rientjes3f36d862019-10-14 14:12:04 -07004702 * order, fail immediately unless the allocator has
4703 * requested compaction and reclaim retry.
David Rientjesb39d0ee2019-09-04 12:54:22 -07004704 *
4705 * Reclaim is
4706 * - potentially very expensive because zones are far
4707 * below their low watermarks or this is part of very
4708 * bursty high order allocations,
4709 * - not guaranteed to help because isolate_freepages()
4710 * may not iterate over freed pages as part of its
4711 * linear scan, and
4712 * - unlikely to make entire pageblocks free on its
4713 * own.
4714 */
4715 if (compact_result == COMPACT_SKIPPED ||
4716 compact_result == COMPACT_DEFERRED)
4717 goto nopage;
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004718
4719 /*
Vlastimil Babka3eb27712016-07-28 15:49:22 -07004720 * Looks like reclaim/compaction is worth trying, but
4721 * sync compaction could be very expensive, so keep
Vlastimil Babka25160352016-07-28 15:49:25 -07004722 * using async compaction.
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004723 */
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004724 compact_priority = INIT_COMPACT_PRIORITY;
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004725 }
4726 }
Vlastimil Babka23771232016-07-28 15:49:16 -07004727
4728retry:
4729 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
Mel Gorman0a79cda2018-12-28 00:35:48 -08004730 if (alloc_flags & ALLOC_KSWAPD)
David Rientjes5ecd9d42018-04-05 16:25:16 -07004731 wake_all_kswapds(order, gfp_mask, ac);
Vlastimil Babka23771232016-07-28 15:49:16 -07004732
Michal Hockocd04ae12017-09-06 16:24:50 -07004733 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4734 if (reserve_flags)
Joonsoo Kim8510e692020-08-06 23:26:04 -07004735 alloc_flags = current_alloc_flags(gfp_mask, reserve_flags);
Vlastimil Babka23771232016-07-28 15:49:16 -07004736
4737 /*
Vlastimil Babkad6a24df2018-08-17 15:45:05 -07004738 * Reset the nodemask and zonelist iterators if memory policies can be
4739 * ignored. These allocations are high priority and system rather than
4740 * user oriented.
Mel Gormane46e7b72016-06-03 14:56:01 -07004741 */
Michal Hockocd04ae12017-09-06 16:24:50 -07004742 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
Vlastimil Babkad6a24df2018-08-17 15:45:05 -07004743 ac->nodemask = NULL;
Mel Gormane46e7b72016-06-03 14:56:01 -07004744 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004745 ac->highest_zoneidx, ac->nodemask);
Mel Gormane46e7b72016-06-03 14:56:01 -07004746 }
4747
Vlastimil Babka23771232016-07-28 15:49:16 -07004748 /* Attempt with potentially adjusted zonelist and alloc_flags */
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004749 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004750 if (page)
4751 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004752
Mel Gormand0164ad2015-11-06 16:28:21 -08004753 /* Caller is not willing to reclaim, we can't balance anything */
Michal Hocko9a67f642017-02-22 15:46:19 -08004754 if (!can_direct_reclaim)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004755 goto nopage;
Michal Hocko9a67f642017-02-22 15:46:19 -08004756
Peter Zijlstra341ce062009-06-16 15:32:02 -07004757 /* Avoid recursion of direct reclaim */
Michal Hocko9a67f642017-02-22 15:46:19 -08004758 if (current->flags & PF_MEMALLOC)
Peter Zijlstra341ce062009-06-16 15:32:02 -07004759 goto nopage;
David Rientjes8fe78042014-08-06 16:07:54 -07004760
Mel Gorman11e33f62009-06-16 15:31:57 -07004761 /* Try direct reclaim and then allocating */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004762 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4763 &did_some_progress);
Mel Gorman11e33f62009-06-16 15:31:57 -07004764 if (page)
4765 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004766
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004767 /* Try direct compaction and then allocating */
4768 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004769 compact_priority, &compact_result);
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004770 if (page)
4771 goto got_pg;
4772
Johannes Weiner90839052015-06-24 16:57:21 -07004773 /* Do not loop if specifically requested */
4774 if (gfp_mask & __GFP_NORETRY)
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004775 goto nopage;
Johannes Weiner90839052015-06-24 16:57:21 -07004776
Michal Hocko0a0337e2016-05-20 16:57:00 -07004777 /*
4778 * Do not retry costly high order allocations unless they are
Michal Hockodcda9b02017-07-12 14:36:45 -07004779 * __GFP_RETRY_MAYFAIL
Michal Hocko0a0337e2016-05-20 16:57:00 -07004780 */
Michal Hockodcda9b02017-07-12 14:36:45 -07004781 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004782 goto nopage;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004783
Michal Hocko0a0337e2016-05-20 16:57:00 -07004784 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07004785 did_some_progress > 0, &no_progress_loops))
Michal Hocko0a0337e2016-05-20 16:57:00 -07004786 goto retry;
4787
Michal Hocko33c2d212016-05-20 16:57:06 -07004788 /*
4789 * It doesn't make any sense to retry for the compaction if the order-0
4790 * reclaim is not able to make any progress because the current
4791 * implementation of the compaction depends on the sufficient amount
4792 * of free memory (see __compaction_suitable)
4793 */
4794 if (did_some_progress > 0 &&
Michal Hocko86a294a2016-05-20 16:57:12 -07004795 should_compact_retry(ac, order, alloc_flags,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004796 compact_result, &compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004797 &compaction_retries))
Michal Hocko33c2d212016-05-20 16:57:06 -07004798 goto retry;
4799
Vlastimil Babka902b6282017-07-06 15:39:56 -07004800
4801 /* Deal with possible cpuset update races before we start OOM killing */
4802 if (check_retry_cpuset(cpuset_mems_cookie, ac))
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004803 goto retry_cpuset;
4804
Johannes Weiner90839052015-06-24 16:57:21 -07004805 /* Reclaim has failed us, start killing things */
4806 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4807 if (page)
4808 goto got_pg;
4809
Michal Hocko9a67f642017-02-22 15:46:19 -08004810 /* Avoid allocations with no watermarks from looping endlessly */
Michal Hockocd04ae12017-09-06 16:24:50 -07004811 if (tsk_is_oom_victim(current) &&
Joonsoo Kim8510e692020-08-06 23:26:04 -07004812 (alloc_flags & ALLOC_OOM ||
Tetsuo Handac2889832017-06-02 14:46:31 -07004813 (gfp_mask & __GFP_NOMEMALLOC)))
Michal Hocko9a67f642017-02-22 15:46:19 -08004814 goto nopage;
4815
Johannes Weiner90839052015-06-24 16:57:21 -07004816 /* Retry as long as the OOM killer is making progress */
Michal Hocko0a0337e2016-05-20 16:57:00 -07004817 if (did_some_progress) {
4818 no_progress_loops = 0;
Johannes Weiner90839052015-06-24 16:57:21 -07004819 goto retry;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004820 }
Johannes Weiner90839052015-06-24 16:57:21 -07004821
Linus Torvalds1da177e2005-04-16 15:20:36 -07004822nopage:
Vlastimil Babka902b6282017-07-06 15:39:56 -07004823 /* Deal with possible cpuset update races before we fail */
4824 if (check_retry_cpuset(cpuset_mems_cookie, ac))
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004825 goto retry_cpuset;
4826
Michal Hocko9a67f642017-02-22 15:46:19 -08004827 /*
4828 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
4829 * we always retry
4830 */
4831 if (gfp_mask & __GFP_NOFAIL) {
4832 /*
4833 * All existing users of the __GFP_NOFAIL are blockable, so warn
4834 * of any new users that actually require GFP_NOWAIT
4835 */
4836 if (WARN_ON_ONCE(!can_direct_reclaim))
4837 goto fail;
4838
4839 /*
4840 * PF_MEMALLOC request from this context is rather bizarre
4841 * because we cannot reclaim anything and only can loop waiting
4842 * for somebody to do a work for us
4843 */
4844 WARN_ON_ONCE(current->flags & PF_MEMALLOC);
4845
4846 /*
4847 * non failing costly orders are a hard requirement which we
4848 * are not prepared for much so let's warn about these users
4849 * so that we can identify them and convert them to something
4850 * else.
4851 */
4852 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
4853
Michal Hocko6c18ba72017-02-22 15:46:25 -08004854 /*
4855 * Help non-failing allocations by giving them access to memory
4856 * reserves but do not use ALLOC_NO_WATERMARKS because this
4857 * could deplete whole memory reserves which would just make
4858 * the situation worse
4859 */
4860 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
4861 if (page)
4862 goto got_pg;
4863
Michal Hocko9a67f642017-02-22 15:46:19 -08004864 cond_resched();
4865 goto retry;
4866 }
4867fail:
Michal Hockoa8e99252017-02-22 15:46:10 -08004868 warn_alloc(gfp_mask, ac->nodemask,
Michal Hocko7877cdc2016-10-07 17:01:55 -07004869 "page allocation failure: order:%u", order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004870got_pg:
Mel Gorman072bb0a2012-07-31 16:43:58 -07004871 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004872}
Mel Gorman11e33f62009-06-16 15:31:57 -07004873
Mel Gorman9cd75552017-02-24 14:56:29 -08004874static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004875 int preferred_nid, nodemask_t *nodemask,
Mel Gorman9cd75552017-02-24 14:56:29 -08004876 struct alloc_context *ac, gfp_t *alloc_mask,
4877 unsigned int *alloc_flags)
4878{
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004879 ac->highest_zoneidx = gfp_zone(gfp_mask);
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004880 ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08004881 ac->nodemask = nodemask;
Wei Yang01c0bfe2020-06-03 15:59:08 -07004882 ac->migratetype = gfp_migratetype(gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08004883
4884 if (cpusets_enabled()) {
4885 *alloc_mask |= __GFP_HARDWALL;
Muchun Song182f3d72020-08-06 23:26:01 -07004886 /*
4887 * When we are in the interrupt context, it is irrelevant
4888 * to the current task context. It means that any node ok.
4889 */
4890 if (!in_interrupt() && !ac->nodemask)
Mel Gorman9cd75552017-02-24 14:56:29 -08004891 ac->nodemask = &cpuset_current_mems_allowed;
Vlastimil Babka51047822017-02-24 14:56:53 -08004892 else
4893 *alloc_flags |= ALLOC_CPUSET;
Mel Gorman9cd75552017-02-24 14:56:29 -08004894 }
4895
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004896 fs_reclaim_acquire(gfp_mask);
4897 fs_reclaim_release(gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08004898
4899 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4900
4901 if (should_fail_alloc_page(gfp_mask, order))
4902 return false;
4903
Joonsoo Kim8510e692020-08-06 23:26:04 -07004904 *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09004905
Mel Gorman9cd75552017-02-24 14:56:29 -08004906 /* Dirty zone balancing only done in the fast path */
4907 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4908
4909 /*
4910 * The preferred zone is used for statistics but crucially it is
4911 * also used as the starting point for the zonelist iterator. It
4912 * may get reset for allocations that ignore memory policies.
4913 */
4914 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004915 ac->highest_zoneidx, ac->nodemask);
Mateusz Noseka0622d02020-10-13 16:55:51 -07004916
4917 return true;
Mel Gorman9cd75552017-02-24 14:56:29 -08004918}
4919
Mel Gorman11e33f62009-06-16 15:31:57 -07004920/*
4921 * This is the 'heart' of the zoned buddy allocator.
4922 */
4923struct page *
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004924__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
4925 nodemask_t *nodemask)
Mel Gorman11e33f62009-06-16 15:31:57 -07004926{
Mel Gorman5bb1b162016-05-19 17:13:50 -07004927 struct page *page;
Mel Gormane6cbd7f2016-07-28 15:46:50 -07004928 unsigned int alloc_flags = ALLOC_WMARK_LOW;
Tetsuo Handaf19360f2017-09-08 16:13:22 -07004929 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
Mel Gorman9cd75552017-02-24 14:56:29 -08004930 struct alloc_context ac = { };
Mel Gorman682a3382016-05-19 17:13:30 -07004931
Michal Hockoc63ae432018-11-16 15:08:53 -08004932 /*
4933 * There are several places where we assume that the order value is sane
4934 * so bail out early if the request is out of bound.
4935 */
4936 if (unlikely(order >= MAX_ORDER)) {
4937 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
4938 return NULL;
4939 }
4940
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +10004941 gfp_mask &= gfp_allowed_mask;
Tetsuo Handaf19360f2017-09-08 16:13:22 -07004942 alloc_mask = gfp_mask;
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004943 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
Mel Gorman11e33f62009-06-16 15:31:57 -07004944 return NULL;
4945
Mel Gorman6bb15452018-12-28 00:35:41 -08004946 /*
4947 * Forbid the first pass from falling back to types that fragment
4948 * memory until all local zones are considered.
4949 */
Mel Gorman0a79cda2018-12-28 00:35:48 -08004950 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);
Mel Gorman6bb15452018-12-28 00:35:41 -08004951
Mel Gorman5117f452009-06-16 15:31:59 -07004952 /* First allocation attempt */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004953 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
Mel Gorman4fcb0972016-05-19 17:14:01 -07004954 if (likely(page))
4955 goto out;
Andrew Morton91fbdc02015-02-11 15:25:04 -08004956
Mel Gorman4fcb0972016-05-19 17:14:01 -07004957 /*
Michal Hocko7dea19f2017-05-03 14:53:15 -07004958 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
4959 * resp. GFP_NOIO which has to be inherited for all allocation requests
4960 * from a particular context which has been marked by
4961 * memalloc_no{fs,io}_{save,restore}.
Mel Gorman4fcb0972016-05-19 17:14:01 -07004962 */
Michal Hocko7dea19f2017-05-03 14:53:15 -07004963 alloc_mask = current_gfp_context(gfp_mask);
Mel Gorman4fcb0972016-05-19 17:14:01 -07004964 ac.spread_dirty_pages = false;
Mel Gorman11e33f62009-06-16 15:31:57 -07004965
Mel Gorman47415262016-05-19 17:14:44 -07004966 /*
4967 * Restore the original nodemask if it was potentially replaced with
4968 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
4969 */
Mateusz Nosek97ce86f2020-04-01 21:09:53 -07004970 ac.nodemask = nodemask;
Vlastimil Babka16096c22017-01-24 15:18:35 -08004971
Mel Gorman4fcb0972016-05-19 17:14:01 -07004972 page = __alloc_pages_slowpath(alloc_mask, order, &ac);
Xishi Qiu23f086f2015-02-11 15:25:07 -08004973
Mel Gorman4fcb0972016-05-19 17:14:01 -07004974out:
Vladimir Davydovc4159a72016-08-08 23:03:12 +03004975 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07004976 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) {
Vladimir Davydovc4159a72016-08-08 23:03:12 +03004977 __free_pages(page, order);
4978 page = NULL;
Vladimir Davydov49491482016-07-26 15:24:24 -07004979 }
4980
Mel Gorman4fcb0972016-05-19 17:14:01 -07004981 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
4982
Mel Gorman11e33f62009-06-16 15:31:57 -07004983 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004984}
Mel Gormand2391712009-06-16 15:31:52 -07004985EXPORT_SYMBOL(__alloc_pages_nodemask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004986
4987/*
Michal Hocko9ea9a682018-08-17 15:46:01 -07004988 * Common helper functions. Never use with __GFP_HIGHMEM because the returned
4989 * address cannot represent highmem pages. Use alloc_pages and then kmap if
4990 * you need to access high mem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004991 */
Harvey Harrison920c7a52008-02-04 22:29:26 -08004992unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004993{
Akinobu Mita945a1112009-09-21 17:01:47 -07004994 struct page *page;
4995
Michal Hocko9ea9a682018-08-17 15:46:01 -07004996 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004997 if (!page)
4998 return 0;
4999 return (unsigned long) page_address(page);
5000}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005001EXPORT_SYMBOL(__get_free_pages);
5002
Harvey Harrison920c7a52008-02-04 22:29:26 -08005003unsigned long get_zeroed_page(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005004{
Akinobu Mita945a1112009-09-21 17:01:47 -07005005 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005006}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005007EXPORT_SYMBOL(get_zeroed_page);
5008
Aaron Lu742aa7f2018-12-28 00:35:22 -08005009static inline void free_the_page(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005010{
Aaron Lu742aa7f2018-12-28 00:35:22 -08005011 if (order == 0) /* Via pcp? */
5012 free_unref_page(page);
5013 else
David Hildenbrand7fef4312020-10-15 20:09:35 -07005014 __free_pages_ok(page, order, FPI_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005015}
5016
Aaron Lu742aa7f2018-12-28 00:35:22 -08005017void __free_pages(struct page *page, unsigned int order)
5018{
5019 if (put_page_testzero(page))
5020 free_the_page(page, order);
Matthew Wilcox (Oracle)e320d302020-10-13 16:56:04 -07005021 else if (!PageHead(page))
5022 while (order-- > 0)
5023 free_the_page(page + (1 << order), order);
Aaron Lu742aa7f2018-12-28 00:35:22 -08005024}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005025EXPORT_SYMBOL(__free_pages);
5026
Harvey Harrison920c7a52008-02-04 22:29:26 -08005027void free_pages(unsigned long addr, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005028{
5029 if (addr != 0) {
Nick Piggin725d7042006-09-25 23:30:55 -07005030 VM_BUG_ON(!virt_addr_valid((void *)addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005031 __free_pages(virt_to_page((void *)addr), order);
5032 }
5033}
5034
5035EXPORT_SYMBOL(free_pages);
5036
Glauber Costa6a1a0d32012-12-18 14:22:00 -08005037/*
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005038 * Page Fragment:
5039 * An arbitrary-length arbitrary-offset area of memory which resides
5040 * within a 0 or higher order page. Multiple fragments within that page
5041 * are individually refcounted, in the page's reference counter.
5042 *
5043 * The page_frag functions below provide a simple allocation framework for
5044 * page fragments. This is used by the network stack and network device
5045 * drivers to provide a backing region of memory for use as either an
5046 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
5047 */
Alexander Duyck2976db82017-01-10 16:58:09 -08005048static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
5049 gfp_t gfp_mask)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005050{
5051 struct page *page = NULL;
5052 gfp_t gfp = gfp_mask;
5053
5054#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5055 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
5056 __GFP_NOMEMALLOC;
5057 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
5058 PAGE_FRAG_CACHE_MAX_ORDER);
5059 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
5060#endif
5061 if (unlikely(!page))
5062 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
5063
5064 nc->va = page ? page_address(page) : NULL;
5065
5066 return page;
5067}
5068
Alexander Duyck2976db82017-01-10 16:58:09 -08005069void __page_frag_cache_drain(struct page *page, unsigned int count)
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005070{
5071 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
5072
Aaron Lu742aa7f2018-12-28 00:35:22 -08005073 if (page_ref_sub_and_test(page, count))
5074 free_the_page(page, compound_order(page));
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005075}
Alexander Duyck2976db82017-01-10 16:58:09 -08005076EXPORT_SYMBOL(__page_frag_cache_drain);
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005077
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005078void *page_frag_alloc(struct page_frag_cache *nc,
5079 unsigned int fragsz, gfp_t gfp_mask)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005080{
5081 unsigned int size = PAGE_SIZE;
5082 struct page *page;
5083 int offset;
5084
5085 if (unlikely(!nc->va)) {
5086refill:
Alexander Duyck2976db82017-01-10 16:58:09 -08005087 page = __page_frag_cache_refill(nc, gfp_mask);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005088 if (!page)
5089 return NULL;
5090
5091#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5092 /* if size can vary use size else just use PAGE_SIZE */
5093 size = nc->size;
5094#endif
5095 /* Even if we own the page, we do not use atomic_set().
5096 * This would break get_page_unless_zero() users.
5097 */
Alexander Duyck86447722019-02-15 14:44:12 -08005098 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005099
5100 /* reset page count bias and offset to start of new frag */
Michal Hocko2f064f32015-08-21 14:11:51 -07005101 nc->pfmemalloc = page_is_pfmemalloc(page);
Alexander Duyck86447722019-02-15 14:44:12 -08005102 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005103 nc->offset = size;
5104 }
5105
5106 offset = nc->offset - fragsz;
5107 if (unlikely(offset < 0)) {
5108 page = virt_to_page(nc->va);
5109
Joonsoo Kimfe896d12016-03-17 14:19:26 -07005110 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005111 goto refill;
5112
Dongli Zhangd8c19012020-11-15 12:10:29 -08005113 if (unlikely(nc->pfmemalloc)) {
5114 free_the_page(page, compound_order(page));
5115 goto refill;
5116 }
5117
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005118#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5119 /* if size can vary use size else just use PAGE_SIZE */
5120 size = nc->size;
5121#endif
5122 /* OK, page count is 0, we can safely set it */
Alexander Duyck86447722019-02-15 14:44:12 -08005123 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005124
5125 /* reset page count bias and offset to start of new frag */
Alexander Duyck86447722019-02-15 14:44:12 -08005126 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005127 offset = size - fragsz;
5128 }
5129
5130 nc->pagecnt_bias--;
5131 nc->offset = offset;
5132
5133 return nc->va + offset;
5134}
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005135EXPORT_SYMBOL(page_frag_alloc);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005136
5137/*
5138 * Frees a page fragment allocated out of either a compound or order 0 page.
5139 */
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005140void page_frag_free(void *addr)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005141{
5142 struct page *page = virt_to_head_page(addr);
5143
Aaron Lu742aa7f2018-12-28 00:35:22 -08005144 if (unlikely(put_page_testzero(page)))
5145 free_the_page(page, compound_order(page));
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005146}
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005147EXPORT_SYMBOL(page_frag_free);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005148
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005149static void *make_alloc_exact(unsigned long addr, unsigned int order,
5150 size_t size)
Andi Kleenee85c2e2011-05-11 15:13:34 -07005151{
5152 if (addr) {
5153 unsigned long alloc_end = addr + (PAGE_SIZE << order);
5154 unsigned long used = addr + PAGE_ALIGN(size);
5155
5156 split_page(virt_to_page((void *)addr), order);
5157 while (used < alloc_end) {
5158 free_page(used);
5159 used += PAGE_SIZE;
5160 }
5161 }
5162 return (void *)addr;
5163}
5164
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005165/**
5166 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
5167 * @size: the number of bytes to allocate
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005168 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005169 *
5170 * This function is similar to alloc_pages(), except that it allocates the
5171 * minimum number of pages to satisfy the request. alloc_pages() can only
5172 * allocate memory in power-of-two pages.
5173 *
5174 * This function is also limited by MAX_ORDER.
5175 *
5176 * Memory allocated by this function must be released by free_pages_exact().
Mike Rapoporta862f682019-03-05 15:48:42 -08005177 *
5178 * Return: pointer to the allocated area or %NULL in case of error.
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005179 */
5180void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
5181{
5182 unsigned int order = get_order(size);
5183 unsigned long addr;
5184
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005185 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
5186 gfp_mask &= ~__GFP_COMP;
5187
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005188 addr = __get_free_pages(gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07005189 return make_alloc_exact(addr, order, size);
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005190}
5191EXPORT_SYMBOL(alloc_pages_exact);
5192
5193/**
Andi Kleenee85c2e2011-05-11 15:13:34 -07005194 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
5195 * pages on a node.
Randy Dunlapb5e6ab52011-05-16 13:16:54 -07005196 * @nid: the preferred node ID where memory should be allocated
Andi Kleenee85c2e2011-05-11 15:13:34 -07005197 * @size: the number of bytes to allocate
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005198 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
Andi Kleenee85c2e2011-05-11 15:13:34 -07005199 *
5200 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
5201 * back.
Mike Rapoporta862f682019-03-05 15:48:42 -08005202 *
5203 * Return: pointer to the allocated area or %NULL in case of error.
Andi Kleenee85c2e2011-05-11 15:13:34 -07005204 */
Fabian Fredericke1931812014-08-06 16:04:59 -07005205void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
Andi Kleenee85c2e2011-05-11 15:13:34 -07005206{
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005207 unsigned int order = get_order(size);
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005208 struct page *p;
5209
5210 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
5211 gfp_mask &= ~__GFP_COMP;
5212
5213 p = alloc_pages_node(nid, gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07005214 if (!p)
5215 return NULL;
5216 return make_alloc_exact((unsigned long)page_address(p), order, size);
5217}
Andi Kleenee85c2e2011-05-11 15:13:34 -07005218
5219/**
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005220 * free_pages_exact - release memory allocated via alloc_pages_exact()
5221 * @virt: the value returned by alloc_pages_exact.
5222 * @size: size of allocation, same value as passed to alloc_pages_exact().
5223 *
5224 * Release the memory allocated by a previous call to alloc_pages_exact.
5225 */
5226void free_pages_exact(void *virt, size_t size)
5227{
5228 unsigned long addr = (unsigned long)virt;
5229 unsigned long end = addr + PAGE_ALIGN(size);
5230
5231 while (addr < end) {
5232 free_page(addr);
5233 addr += PAGE_SIZE;
5234 }
5235}
5236EXPORT_SYMBOL(free_pages_exact);
5237
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005238/**
5239 * nr_free_zone_pages - count number of pages beyond high watermark
5240 * @offset: The zone index of the highest zone
5241 *
Mike Rapoporta862f682019-03-05 15:48:42 -08005242 * nr_free_zone_pages() counts the number of pages which are beyond the
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005243 * high watermark within all zones at or below a given zone index. For each
5244 * zone, the number of pages is calculated as:
mchehab@s-opensource.com0e056eb2017-03-30 17:11:36 -03005245 *
5246 * nr_free_zone_pages = managed_pages - high_pages
Mike Rapoporta862f682019-03-05 15:48:42 -08005247 *
5248 * Return: number of pages beyond high watermark.
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005249 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005250static unsigned long nr_free_zone_pages(int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005251{
Mel Gormandd1a2392008-04-28 02:12:17 -07005252 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07005253 struct zone *zone;
5254
Martin J. Blighe310fd42005-07-29 22:59:18 -07005255 /* Just pick one node, since fallback list is circular */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005256 unsigned long sum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005257
Mel Gorman0e884602008-04-28 02:12:14 -07005258 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005259
Mel Gorman54a6eb52008-04-28 02:12:16 -07005260 for_each_zone_zonelist(zone, z, zonelist, offset) {
Arun KS9705bea2018-12-28 00:34:24 -08005261 unsigned long size = zone_managed_pages(zone);
Mel Gorman41858962009-06-16 15:32:12 -07005262 unsigned long high = high_wmark_pages(zone);
Martin J. Blighe310fd42005-07-29 22:59:18 -07005263 if (size > high)
5264 sum += size - high;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005265 }
5266
5267 return sum;
5268}
5269
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005270/**
5271 * nr_free_buffer_pages - count number of pages beyond high watermark
5272 *
5273 * nr_free_buffer_pages() counts the number of pages which are beyond the high
5274 * watermark within ZONE_DMA and ZONE_NORMAL.
Mike Rapoporta862f682019-03-05 15:48:42 -08005275 *
5276 * Return: number of pages beyond high watermark within ZONE_DMA and
5277 * ZONE_NORMAL.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005278 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005279unsigned long nr_free_buffer_pages(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005280{
Al Viroaf4ca452005-10-21 02:55:38 -04005281 return nr_free_zone_pages(gfp_zone(GFP_USER));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005282}
Meelap Shahc2f1a552007-07-17 04:04:39 -07005283EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005284
Christoph Lameter08e0f6a2006-09-27 01:50:06 -07005285static inline void show_node(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005286{
Kirill A. Shutemove5adfff2012-12-11 16:00:29 -08005287 if (IS_ENABLED(CONFIG_NUMA))
Andy Whitcroft25ba77c2006-12-06 20:33:03 -08005288 printk("Node %d ", zone_to_nid(zone));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005289}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005290
Igor Redkod02bd272016-03-17 14:19:05 -07005291long si_mem_available(void)
5292{
5293 long available;
5294 unsigned long pagecache;
5295 unsigned long wmark_low = 0;
5296 unsigned long pages[NR_LRU_LISTS];
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005297 unsigned long reclaimable;
Igor Redkod02bd272016-03-17 14:19:05 -07005298 struct zone *zone;
5299 int lru;
5300
5301 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
Mel Gorman2f95ff92016-08-11 15:32:57 -07005302 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
Igor Redkod02bd272016-03-17 14:19:05 -07005303
5304 for_each_zone(zone)
Mel Gormana9214442018-12-28 00:35:44 -08005305 wmark_low += low_wmark_pages(zone);
Igor Redkod02bd272016-03-17 14:19:05 -07005306
5307 /*
5308 * Estimate the amount of memory available for userspace allocations,
5309 * without causing swapping.
5310 */
Michal Hockoc41f0122017-09-06 16:23:36 -07005311 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
Igor Redkod02bd272016-03-17 14:19:05 -07005312
5313 /*
5314 * Not all the page cache can be freed, otherwise the system will
5315 * start swapping. Assume at least half of the page cache, or the
5316 * low watermark worth of cache, needs to stay.
5317 */
5318 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
5319 pagecache -= min(pagecache / 2, wmark_low);
5320 available += pagecache;
5321
5322 /*
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005323 * Part of the reclaimable slab and other kernel memory consists of
5324 * items that are in use, and cannot be freed. Cap this estimate at the
5325 * low watermark.
Igor Redkod02bd272016-03-17 14:19:05 -07005326 */
Roman Gushchind42f3242020-08-06 23:20:39 -07005327 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) +
5328 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005329 available += reclaimable - min(reclaimable / 2, wmark_low);
Roman Gushchin034ebf62018-04-10 16:27:40 -07005330
Igor Redkod02bd272016-03-17 14:19:05 -07005331 if (available < 0)
5332 available = 0;
5333 return available;
5334}
5335EXPORT_SYMBOL_GPL(si_mem_available);
5336
Linus Torvalds1da177e2005-04-16 15:20:36 -07005337void si_meminfo(struct sysinfo *val)
5338{
Arun KSca79b0c2018-12-28 00:34:29 -08005339 val->totalram = totalram_pages();
Mel Gorman11fb9982016-07-28 15:46:20 -07005340 val->sharedram = global_node_page_state(NR_SHMEM);
Michal Hockoc41f0122017-09-06 16:23:36 -07005341 val->freeram = global_zone_page_state(NR_FREE_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005342 val->bufferram = nr_blockdev_pages();
Arun KSca79b0c2018-12-28 00:34:29 -08005343 val->totalhigh = totalhigh_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005344 val->freehigh = nr_free_highpages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005345 val->mem_unit = PAGE_SIZE;
5346}
5347
5348EXPORT_SYMBOL(si_meminfo);
5349
5350#ifdef CONFIG_NUMA
5351void si_meminfo_node(struct sysinfo *val, int nid)
5352{
Jiang Liucdd91a72013-07-03 15:03:27 -07005353 int zone_type; /* needs to be signed */
5354 unsigned long managed_pages = 0;
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005355 unsigned long managed_highpages = 0;
5356 unsigned long free_highpages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005357 pg_data_t *pgdat = NODE_DATA(nid);
5358
Jiang Liucdd91a72013-07-03 15:03:27 -07005359 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
Arun KS9705bea2018-12-28 00:34:24 -08005360 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
Jiang Liucdd91a72013-07-03 15:03:27 -07005361 val->totalram = managed_pages;
Mel Gorman11fb9982016-07-28 15:46:20 -07005362 val->sharedram = node_page_state(pgdat, NR_SHMEM);
Mel Gorman75ef7182016-07-28 15:45:24 -07005363 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005364#ifdef CONFIG_HIGHMEM
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005365 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
5366 struct zone *zone = &pgdat->node_zones[zone_type];
5367
5368 if (is_highmem(zone)) {
Arun KS9705bea2018-12-28 00:34:24 -08005369 managed_highpages += zone_managed_pages(zone);
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005370 free_highpages += zone_page_state(zone, NR_FREE_PAGES);
5371 }
5372 }
5373 val->totalhigh = managed_highpages;
5374 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005375#else
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005376 val->totalhigh = managed_highpages;
5377 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005378#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005379 val->mem_unit = PAGE_SIZE;
5380}
5381#endif
5382
David Rientjesddd588b2011-03-22 16:30:46 -07005383/*
David Rientjes7bf02ea2011-05-24 17:11:16 -07005384 * Determine whether the node should be displayed or not, depending on whether
5385 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
David Rientjesddd588b2011-03-22 16:30:46 -07005386 */
Michal Hocko9af744d2017-02-22 15:46:16 -08005387static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
David Rientjesddd588b2011-03-22 16:30:46 -07005388{
David Rientjesddd588b2011-03-22 16:30:46 -07005389 if (!(flags & SHOW_MEM_FILTER_NODES))
Michal Hocko9af744d2017-02-22 15:46:16 -08005390 return false;
David Rientjesddd588b2011-03-22 16:30:46 -07005391
Michal Hocko9af744d2017-02-22 15:46:16 -08005392 /*
5393 * no node mask - aka implicit memory numa policy. Do not bother with
5394 * the synchronization - read_mems_allowed_begin - because we do not
5395 * have to be precise here.
5396 */
5397 if (!nodemask)
5398 nodemask = &cpuset_current_mems_allowed;
5399
5400 return !node_isset(nid, *nodemask);
David Rientjesddd588b2011-03-22 16:30:46 -07005401}
5402
Linus Torvalds1da177e2005-04-16 15:20:36 -07005403#define K(x) ((x) << (PAGE_SHIFT-10))
5404
Rabin Vincent377e4f12012-12-11 16:00:24 -08005405static void show_migration_types(unsigned char type)
5406{
5407 static const char types[MIGRATE_TYPES] = {
5408 [MIGRATE_UNMOVABLE] = 'U',
Rabin Vincent377e4f12012-12-11 16:00:24 -08005409 [MIGRATE_MOVABLE] = 'M',
Vlastimil Babka475a2f92015-12-11 13:40:29 -08005410 [MIGRATE_RECLAIMABLE] = 'E',
5411 [MIGRATE_HIGHATOMIC] = 'H',
Rabin Vincent377e4f12012-12-11 16:00:24 -08005412#ifdef CONFIG_CMA
5413 [MIGRATE_CMA] = 'C',
5414#endif
Minchan Kim194159f2013-02-22 16:33:58 -08005415#ifdef CONFIG_MEMORY_ISOLATION
Rabin Vincent377e4f12012-12-11 16:00:24 -08005416 [MIGRATE_ISOLATE] = 'I',
Minchan Kim194159f2013-02-22 16:33:58 -08005417#endif
Rabin Vincent377e4f12012-12-11 16:00:24 -08005418 };
5419 char tmp[MIGRATE_TYPES + 1];
5420 char *p = tmp;
5421 int i;
5422
5423 for (i = 0; i < MIGRATE_TYPES; i++) {
5424 if (type & (1 << i))
5425 *p++ = types[i];
5426 }
5427
5428 *p = '\0';
Joe Perches1f84a182016-10-27 17:46:29 -07005429 printk(KERN_CONT "(%s) ", tmp);
Rabin Vincent377e4f12012-12-11 16:00:24 -08005430}
5431
Linus Torvalds1da177e2005-04-16 15:20:36 -07005432/*
5433 * Show free area list (used inside shift_scroll-lock stuff)
5434 * We also calculate the percentage fragmentation. We do this by counting the
5435 * memory on each free list with the exception of the first item on the list.
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005436 *
5437 * Bits in @filter:
5438 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
5439 * cpuset.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005440 */
Michal Hocko9af744d2017-02-22 15:46:16 -08005441void show_free_areas(unsigned int filter, nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005442{
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005443 unsigned long free_pcp = 0;
Jes Sorensenc7241912006-09-27 01:50:05 -07005444 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005445 struct zone *zone;
Mel Gorman599d0c92016-07-28 15:45:31 -07005446 pg_data_t *pgdat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005447
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005448 for_each_populated_zone(zone) {
Michal Hocko9af744d2017-02-22 15:46:16 -08005449 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005450 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005451
Konstantin Khlebnikov761b0672015-04-14 15:45:32 -07005452 for_each_online_cpu(cpu)
5453 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005454 }
5455
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07005456 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
5457 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
NeilBrown8d928902020-06-01 21:48:21 -07005458 " unevictable:%lu dirty:%lu writeback:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005459 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07005460 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005461 " free:%lu free_pcp:%lu free_cma:%lu\n",
Mel Gorman599d0c92016-07-28 15:45:31 -07005462 global_node_page_state(NR_ACTIVE_ANON),
5463 global_node_page_state(NR_INACTIVE_ANON),
5464 global_node_page_state(NR_ISOLATED_ANON),
5465 global_node_page_state(NR_ACTIVE_FILE),
5466 global_node_page_state(NR_INACTIVE_FILE),
5467 global_node_page_state(NR_ISOLATED_FILE),
5468 global_node_page_state(NR_UNEVICTABLE),
Mel Gorman11fb9982016-07-28 15:46:20 -07005469 global_node_page_state(NR_FILE_DIRTY),
5470 global_node_page_state(NR_WRITEBACK),
Roman Gushchind42f3242020-08-06 23:20:39 -07005471 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B),
5472 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B),
Mel Gorman50658e22016-07-28 15:46:14 -07005473 global_node_page_state(NR_FILE_MAPPED),
Mel Gorman11fb9982016-07-28 15:46:20 -07005474 global_node_page_state(NR_SHMEM),
Michal Hockoc41f0122017-09-06 16:23:36 -07005475 global_zone_page_state(NR_PAGETABLE),
5476 global_zone_page_state(NR_BOUNCE),
5477 global_zone_page_state(NR_FREE_PAGES),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005478 free_pcp,
Michal Hockoc41f0122017-09-06 16:23:36 -07005479 global_zone_page_state(NR_FREE_CMA_PAGES));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005480
Mel Gorman599d0c92016-07-28 15:45:31 -07005481 for_each_online_pgdat(pgdat) {
Michal Hocko9af744d2017-02-22 15:46:16 -08005482 if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
Michal Hockoc02e50b2017-02-22 15:46:07 -08005483 continue;
5484
Mel Gorman599d0c92016-07-28 15:45:31 -07005485 printk("Node %d"
5486 " active_anon:%lukB"
5487 " inactive_anon:%lukB"
5488 " active_file:%lukB"
5489 " inactive_file:%lukB"
5490 " unevictable:%lukB"
5491 " isolated(anon):%lukB"
5492 " isolated(file):%lukB"
Mel Gorman50658e22016-07-28 15:46:14 -07005493 " mapped:%lukB"
Mel Gorman11fb9982016-07-28 15:46:20 -07005494 " dirty:%lukB"
5495 " writeback:%lukB"
5496 " shmem:%lukB"
5497#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5498 " shmem_thp: %lukB"
5499 " shmem_pmdmapped: %lukB"
5500 " anon_thp: %lukB"
5501#endif
5502 " writeback_tmp:%lukB"
Shakeel Butt991e7672020-08-06 23:21:37 -07005503 " kernel_stack:%lukB"
5504#ifdef CONFIG_SHADOW_CALL_STACK
5505 " shadow_call_stack:%lukB"
5506#endif
Mel Gorman599d0c92016-07-28 15:45:31 -07005507 " all_unreclaimable? %s"
5508 "\n",
5509 pgdat->node_id,
5510 K(node_page_state(pgdat, NR_ACTIVE_ANON)),
5511 K(node_page_state(pgdat, NR_INACTIVE_ANON)),
5512 K(node_page_state(pgdat, NR_ACTIVE_FILE)),
5513 K(node_page_state(pgdat, NR_INACTIVE_FILE)),
5514 K(node_page_state(pgdat, NR_UNEVICTABLE)),
5515 K(node_page_state(pgdat, NR_ISOLATED_ANON)),
5516 K(node_page_state(pgdat, NR_ISOLATED_FILE)),
Mel Gorman50658e22016-07-28 15:46:14 -07005517 K(node_page_state(pgdat, NR_FILE_MAPPED)),
Mel Gorman11fb9982016-07-28 15:46:20 -07005518 K(node_page_state(pgdat, NR_FILE_DIRTY)),
5519 K(node_page_state(pgdat, NR_WRITEBACK)),
Alexander Polakov1f06b812017-04-07 16:04:45 -07005520 K(node_page_state(pgdat, NR_SHMEM)),
Mel Gorman11fb9982016-07-28 15:46:20 -07005521#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5522 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
5523 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
5524 * HPAGE_PMD_NR),
5525 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
5526#endif
Mel Gorman11fb9982016-07-28 15:46:20 -07005527 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
Shakeel Butt991e7672020-08-06 23:21:37 -07005528 node_page_state(pgdat, NR_KERNEL_STACK_KB),
5529#ifdef CONFIG_SHADOW_CALL_STACK
5530 node_page_state(pgdat, NR_KERNEL_SCS_KB),
5531#endif
Johannes Weinerc73322d2017-05-03 14:51:51 -07005532 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
5533 "yes" : "no");
Mel Gorman599d0c92016-07-28 15:45:31 -07005534 }
5535
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005536 for_each_populated_zone(zone) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005537 int i;
5538
Michal Hocko9af744d2017-02-22 15:46:16 -08005539 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005540 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005541
5542 free_pcp = 0;
5543 for_each_online_cpu(cpu)
5544 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5545
Linus Torvalds1da177e2005-04-16 15:20:36 -07005546 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07005547 printk(KERN_CONT
5548 "%s"
Linus Torvalds1da177e2005-04-16 15:20:36 -07005549 " free:%lukB"
5550 " min:%lukB"
5551 " low:%lukB"
5552 " high:%lukB"
lijiazie47b3462019-11-30 17:55:21 -08005553 " reserved_highatomic:%luKB"
Minchan Kim71c799f2016-07-28 15:47:26 -07005554 " active_anon:%lukB"
5555 " inactive_anon:%lukB"
5556 " active_file:%lukB"
5557 " inactive_file:%lukB"
5558 " unevictable:%lukB"
Mel Gorman5a1c84b2016-07-28 15:47:31 -07005559 " writepending:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07005560 " present:%lukB"
Jiang Liu9feedc92012-12-12 13:52:12 -08005561 " managed:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005562 " mlocked:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005563 " pagetables:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005564 " bounce:%lukB"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005565 " free_pcp:%lukB"
5566 " local_pcp:%ukB"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07005567 " free_cma:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07005568 "\n",
5569 zone->name,
Mel Gorman88f5acf2011-01-13 15:45:41 -08005570 K(zone_page_state(zone, NR_FREE_PAGES)),
Mel Gorman41858962009-06-16 15:32:12 -07005571 K(min_wmark_pages(zone)),
5572 K(low_wmark_pages(zone)),
5573 K(high_wmark_pages(zone)),
lijiazie47b3462019-11-30 17:55:21 -08005574 K(zone->nr_reserved_highatomic),
Minchan Kim71c799f2016-07-28 15:47:26 -07005575 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
5576 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
5577 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
5578 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
5579 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
Mel Gorman5a1c84b2016-07-28 15:47:31 -07005580 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
Linus Torvalds1da177e2005-04-16 15:20:36 -07005581 K(zone->present_pages),
Arun KS9705bea2018-12-28 00:34:24 -08005582 K(zone_managed_pages(zone)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005583 K(zone_page_state(zone, NR_MLOCK)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005584 K(zone_page_state(zone, NR_PAGETABLE)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005585 K(zone_page_state(zone, NR_BOUNCE)),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005586 K(free_pcp),
5587 K(this_cpu_read(zone->pageset->pcp.count)),
Minchan Kim33e077b2016-07-28 15:47:14 -07005588 K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005589 printk("lowmem_reserve[]:");
5590 for (i = 0; i < MAX_NR_ZONES; i++)
Joe Perches1f84a182016-10-27 17:46:29 -07005591 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
5592 printk(KERN_CONT "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005593 }
5594
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005595 for_each_populated_zone(zone) {
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005596 unsigned int order;
5597 unsigned long nr[MAX_ORDER], flags, total = 0;
Rabin Vincent377e4f12012-12-11 16:00:24 -08005598 unsigned char types[MAX_ORDER];
Linus Torvalds1da177e2005-04-16 15:20:36 -07005599
Michal Hocko9af744d2017-02-22 15:46:16 -08005600 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005601 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005602 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07005603 printk(KERN_CONT "%s: ", zone->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005604
5605 spin_lock_irqsave(&zone->lock, flags);
5606 for (order = 0; order < MAX_ORDER; order++) {
Rabin Vincent377e4f12012-12-11 16:00:24 -08005607 struct free_area *area = &zone->free_area[order];
5608 int type;
5609
5610 nr[order] = area->nr_free;
Kirill Korotaev8f9de512006-06-23 02:03:50 -07005611 total += nr[order] << order;
Rabin Vincent377e4f12012-12-11 16:00:24 -08005612
5613 types[order] = 0;
5614 for (type = 0; type < MIGRATE_TYPES; type++) {
Dan Williamsb03641a2019-05-14 15:41:32 -07005615 if (!free_area_empty(area, type))
Rabin Vincent377e4f12012-12-11 16:00:24 -08005616 types[order] |= 1 << type;
5617 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005618 }
5619 spin_unlock_irqrestore(&zone->lock, flags);
Rabin Vincent377e4f12012-12-11 16:00:24 -08005620 for (order = 0; order < MAX_ORDER; order++) {
Joe Perches1f84a182016-10-27 17:46:29 -07005621 printk(KERN_CONT "%lu*%lukB ",
5622 nr[order], K(1UL) << order);
Rabin Vincent377e4f12012-12-11 16:00:24 -08005623 if (nr[order])
5624 show_migration_types(types[order]);
5625 }
Joe Perches1f84a182016-10-27 17:46:29 -07005626 printk(KERN_CONT "= %lukB\n", K(total));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005627 }
5628
David Rientjes949f7ec2013-04-29 15:07:48 -07005629 hugetlb_show_meminfo();
5630
Mel Gorman11fb9982016-07-28 15:46:20 -07005631 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
Larry Woodmane6f36022008-02-04 22:29:30 -08005632
Linus Torvalds1da177e2005-04-16 15:20:36 -07005633 show_swap_cache_info();
5634}
5635
Mel Gorman19770b32008-04-28 02:12:18 -07005636static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
5637{
5638 zoneref->zone = zone;
5639 zoneref->zone_idx = zone_idx(zone);
5640}
5641
Linus Torvalds1da177e2005-04-16 15:20:36 -07005642/*
5643 * Builds allocation fallback zone lists.
Christoph Lameter1a932052006-01-06 00:11:16 -08005644 *
5645 * Add all populated zones of a node to the zonelist.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005646 */
Michal Hocko9d3be212017-09-06 16:20:30 -07005647static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005648{
Christoph Lameter1a932052006-01-06 00:11:16 -08005649 struct zone *zone;
Zhang Yanfeibc732f12013-07-08 16:00:06 -07005650 enum zone_type zone_type = MAX_NR_ZONES;
Michal Hocko9d3be212017-09-06 16:20:30 -07005651 int nr_zones = 0;
Christoph Lameter02a68a52006-01-06 00:11:18 -08005652
5653 do {
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005654 zone_type--;
Christoph Lameter070f8032006-01-06 00:11:19 -08005655 zone = pgdat->node_zones + zone_type;
Juergen Gross192e5072022-04-14 19:13:43 -07005656 if (populated_zone(zone)) {
Michal Hocko9d3be212017-09-06 16:20:30 -07005657 zoneref_set_zone(zone, &zonerefs[nr_zones++]);
Christoph Lameter070f8032006-01-06 00:11:19 -08005658 check_highest_zone(zone_type);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005659 }
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005660 } while (zone_type);
Zhang Yanfeibc732f12013-07-08 16:00:06 -07005661
Christoph Lameter070f8032006-01-06 00:11:19 -08005662 return nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005663}
5664
5665#ifdef CONFIG_NUMA
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005666
5667static int __parse_numa_zonelist_order(char *s)
5668{
Michal Hockoc9bff3e2017-09-06 16:20:13 -07005669 /*
5670 * We used to support different zonlists modes but they turned
5671 * out to be just not useful. Let's keep the warning in place
5672 * if somebody still use the cmd line parameter so that we do
5673 * not fail it silently
5674 */
5675 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
5676 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005677 return -EINVAL;
5678 }
5679 return 0;
5680}
5681
Michal Hockoc9bff3e2017-09-06 16:20:13 -07005682char numa_zonelist_order[] = "Node";
5683
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005684/*
5685 * sysctl handler for numa_zonelist_order
5686 */
Joe Perchescccad5b2014-06-06 14:38:09 -07005687int numa_zonelist_order_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02005688 void *buffer, size_t *length, loff_t *ppos)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005689{
Christoph Hellwig32927392020-04-24 08:43:38 +02005690 if (write)
5691 return __parse_numa_zonelist_order(buffer);
5692 return proc_dostring(table, write, buffer, length, ppos);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005693}
5694
5695
Christoph Lameter62bc62a2009-06-16 15:32:15 -07005696#define MAX_NODE_LOAD (nr_online_nodes)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005697static int node_load[MAX_NUMNODES];
5698
Linus Torvalds1da177e2005-04-16 15:20:36 -07005699/**
Pavel Pisa4dc3b162005-05-01 08:59:25 -07005700 * find_next_best_node - find the next node that should appear in a given node's fallback list
Linus Torvalds1da177e2005-04-16 15:20:36 -07005701 * @node: node whose fallback list we're appending
5702 * @used_node_mask: nodemask_t of already used nodes
5703 *
5704 * We use a number of factors to determine which is the next node that should
5705 * appear on a given node's fallback list. The node should not have appeared
5706 * already in @node's fallback list, and it should be the next closest node
5707 * according to the distance array (which contains arbitrary distance values
5708 * from each node to each node in the system), and should also prefer nodes
5709 * with no CPUs, since presumably they'll have very little allocation pressure
5710 * on them otherwise.
Mike Rapoporta862f682019-03-05 15:48:42 -08005711 *
5712 * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005713 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005714static int find_next_best_node(int node, nodemask_t *used_node_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005715{
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01005716 int n, val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005717 int min_val = INT_MAX;
David Rientjes00ef2d22013-02-22 16:35:36 -08005718 int best_node = NUMA_NO_NODE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005719
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01005720 /* Use the local node if we haven't already */
5721 if (!node_isset(node, *used_node_mask)) {
5722 node_set(node, *used_node_mask);
5723 return node;
5724 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005725
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08005726 for_each_node_state(n, N_MEMORY) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005727
5728 /* Don't want a node to appear more than once */
5729 if (node_isset(n, *used_node_mask))
5730 continue;
5731
Linus Torvalds1da177e2005-04-16 15:20:36 -07005732 /* Use the distance array to find the distance */
5733 val = node_distance(node, n);
5734
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01005735 /* Penalize nodes under us ("prefer the next node") */
5736 val += (n < node);
5737
Linus Torvalds1da177e2005-04-16 15:20:36 -07005738 /* Give preference to headless and unused nodes */
Mateusz Nosekb6307492020-10-13 16:55:42 -07005739 if (!cpumask_empty(cpumask_of_node(n)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005740 val += PENALTY_FOR_NODE_WITH_CPUS;
5741
5742 /* Slight preference for less loaded node */
5743 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
5744 val += node_load[n];
5745
5746 if (val < min_val) {
5747 min_val = val;
5748 best_node = n;
5749 }
5750 }
5751
5752 if (best_node >= 0)
5753 node_set(best_node, *used_node_mask);
5754
5755 return best_node;
5756}
5757
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005758
5759/*
5760 * Build zonelists ordered by node and zones within node.
5761 * This results in maximum locality--normal zone overflows into local
5762 * DMA zone, if any--but risks exhausting DMA zone.
5763 */
Michal Hocko9d3be212017-09-06 16:20:30 -07005764static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
5765 unsigned nr_nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005766{
Michal Hocko9d3be212017-09-06 16:20:30 -07005767 struct zoneref *zonerefs;
5768 int i;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005769
Michal Hocko9d3be212017-09-06 16:20:30 -07005770 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5771
5772 for (i = 0; i < nr_nodes; i++) {
5773 int nr_zones;
5774
5775 pg_data_t *node = NODE_DATA(node_order[i]);
5776
5777 nr_zones = build_zonerefs_node(node, zonerefs);
5778 zonerefs += nr_zones;
5779 }
5780 zonerefs->zone = NULL;
5781 zonerefs->zone_idx = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005782}
5783
5784/*
Christoph Lameter523b9452007-10-16 01:25:37 -07005785 * Build gfp_thisnode zonelists
5786 */
5787static void build_thisnode_zonelists(pg_data_t *pgdat)
5788{
Michal Hocko9d3be212017-09-06 16:20:30 -07005789 struct zoneref *zonerefs;
5790 int nr_zones;
Christoph Lameter523b9452007-10-16 01:25:37 -07005791
Michal Hocko9d3be212017-09-06 16:20:30 -07005792 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
5793 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5794 zonerefs += nr_zones;
5795 zonerefs->zone = NULL;
5796 zonerefs->zone_idx = 0;
Christoph Lameter523b9452007-10-16 01:25:37 -07005797}
5798
5799/*
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005800 * Build zonelists ordered by zone and nodes within zones.
5801 * This results in conserving DMA zone[s] until all Normal memory is
5802 * exhausted, but results in overflowing to remote node while memory
5803 * may still exist in local DMA zone.
5804 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005805
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005806static void build_zonelists(pg_data_t *pgdat)
5807{
Michal Hocko9d3be212017-09-06 16:20:30 -07005808 static int node_order[MAX_NUMNODES];
5809 int node, load, nr_nodes = 0;
Wei Yangd0ddf492020-06-03 15:59:05 -07005810 nodemask_t used_mask = NODE_MASK_NONE;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005811 int local_node, prev_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005812
5813 /* NUMA-aware ordering of nodes */
5814 local_node = pgdat->node_id;
Christoph Lameter62bc62a2009-06-16 15:32:15 -07005815 load = nr_online_nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005816 prev_node = local_node;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005817
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005818 memset(node_order, 0, sizeof(node_order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005819 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
5820 /*
5821 * We don't want to pressure a particular node.
5822 * So adding penalty to the first node in same
5823 * distance group to make it round-robin.
5824 */
David Rientjes957f8222012-10-08 16:33:24 -07005825 if (node_distance(local_node, node) !=
5826 node_distance(local_node, prev_node))
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005827 node_load[node] = load;
5828
Michal Hocko9d3be212017-09-06 16:20:30 -07005829 node_order[nr_nodes++] = node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005830 prev_node = node;
5831 load--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005832 }
Christoph Lameter523b9452007-10-16 01:25:37 -07005833
Michal Hocko9d3be212017-09-06 16:20:30 -07005834 build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
Christoph Lameter523b9452007-10-16 01:25:37 -07005835 build_thisnode_zonelists(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005836}
5837
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005838#ifdef CONFIG_HAVE_MEMORYLESS_NODES
5839/*
5840 * Return node id of node used for "local" allocations.
5841 * I.e., first node id of first zone in arg node's generic zonelist.
5842 * Used for initializing percpu 'numa_mem', which is used primarily
5843 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
5844 */
5845int local_memory_node(int node)
5846{
Mel Gormanc33d6c02016-05-19 17:14:10 -07005847 struct zoneref *z;
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005848
Mel Gormanc33d6c02016-05-19 17:14:10 -07005849 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005850 gfp_zone(GFP_KERNEL),
Mel Gormanc33d6c02016-05-19 17:14:10 -07005851 NULL);
Pavel Tatashinc1093b72018-08-21 21:53:32 -07005852 return zone_to_nid(z->zone);
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005853}
5854#endif
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005855
Joonsoo Kim6423aa82016-08-10 16:27:49 -07005856static void setup_min_unmapped_ratio(void);
5857static void setup_min_slab_ratio(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005858#else /* CONFIG_NUMA */
5859
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005860static void build_zonelists(pg_data_t *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005861{
Christoph Lameter19655d32006-09-25 23:31:19 -07005862 int node, local_node;
Michal Hocko9d3be212017-09-06 16:20:30 -07005863 struct zoneref *zonerefs;
5864 int nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005865
5866 local_node = pgdat->node_id;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005867
Michal Hocko9d3be212017-09-06 16:20:30 -07005868 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5869 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5870 zonerefs += nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005871
Mel Gorman54a6eb52008-04-28 02:12:16 -07005872 /*
5873 * Now we build the zonelist so that it contains the zones
5874 * of all the other nodes.
5875 * We don't want to pressure a particular node, so when
5876 * building the zones for node N, we make sure that the
5877 * zones coming right after the local ones are those from
5878 * node N+1 (modulo N)
5879 */
5880 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
5881 if (!node_online(node))
5882 continue;
Michal Hocko9d3be212017-09-06 16:20:30 -07005883 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5884 zonerefs += nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005885 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07005886 for (node = 0; node < local_node; node++) {
5887 if (!node_online(node))
5888 continue;
Michal Hocko9d3be212017-09-06 16:20:30 -07005889 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5890 zonerefs += nr_zones;
Mel Gorman54a6eb52008-04-28 02:12:16 -07005891 }
5892
Michal Hocko9d3be212017-09-06 16:20:30 -07005893 zonerefs->zone = NULL;
5894 zonerefs->zone_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005895}
5896
5897#endif /* CONFIG_NUMA */
5898
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005899/*
5900 * Boot pageset table. One per cpu which is going to be used for all
5901 * zones and all nodes. The parameters will be set in such a way
5902 * that an item put on a list will immediately be handed over to
5903 * the buddy list. This is safe since pageset manipulation is done
5904 * with interrupts disabled.
5905 *
5906 * The boot_pagesets must be kept even after bootup is complete for
5907 * unused processors and/or zones. They do play a role for bootstrapping
5908 * hotplugged processors.
5909 *
5910 * zoneinfo_show() and maybe other functions do
5911 * not check if the processor is online before following the pageset pointer.
5912 * Other parts of the kernel may not check if the zone is available.
5913 */
5914static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
5915static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
Johannes Weiner385386c2017-07-06 15:40:43 -07005916static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005917
Michal Hocko11cd8632017-09-06 16:20:34 -07005918static void __build_all_zonelists(void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005919{
Yasunori Goto68113782006-06-23 02:03:11 -07005920 int nid;
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005921 int __maybe_unused cpu;
Jiang Liu9adb62a2012-07-31 16:43:28 -07005922 pg_data_t *self = data;
Michal Hockob93e0f32017-09-06 16:20:37 -07005923 static DEFINE_SPINLOCK(lock);
5924
5925 spin_lock(&lock);
Paul Jackson9276b1bc2006-12-06 20:31:48 -08005926
Bo Liu7f9cfb32009-08-18 14:11:19 -07005927#ifdef CONFIG_NUMA
5928 memset(node_load, 0, sizeof(node_load));
5929#endif
Jiang Liu9adb62a2012-07-31 16:43:28 -07005930
Wei Yangc1152582017-09-06 16:19:33 -07005931 /*
5932 * This node is hotadded and no memory is yet present. So just
5933 * building zonelists is fine - no need to touch other nodes.
5934 */
Jiang Liu9adb62a2012-07-31 16:43:28 -07005935 if (self && !node_online(self->node_id)) {
5936 build_zonelists(self);
Wei Yangc1152582017-09-06 16:19:33 -07005937 } else {
5938 for_each_online_node(nid) {
5939 pg_data_t *pgdat = NODE_DATA(nid);
Jiang Liu9adb62a2012-07-31 16:43:28 -07005940
Wei Yangc1152582017-09-06 16:19:33 -07005941 build_zonelists(pgdat);
5942 }
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005943
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005944#ifdef CONFIG_HAVE_MEMORYLESS_NODES
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005945 /*
5946 * We now know the "local memory node" for each node--
5947 * i.e., the node of the first zone in the generic zonelist.
5948 * Set up numa_mem percpu variable for on-line cpus. During
5949 * boot, only the boot cpu should be on-line; we'll init the
5950 * secondary cpus' numa_mem as they come on-line. During
5951 * node/memory hotplug, we'll fixup all on-line cpus.
5952 */
Michal Hockod9c9a0b2017-09-06 16:20:20 -07005953 for_each_online_cpu(cpu)
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005954 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005955#endif
Michal Hockod9c9a0b2017-09-06 16:20:20 -07005956 }
Michal Hockob93e0f32017-09-06 16:20:37 -07005957
5958 spin_unlock(&lock);
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005959}
5960
5961static noinline void __init
5962build_all_zonelists_init(void)
5963{
5964 int cpu;
5965
5966 __build_all_zonelists(NULL);
5967
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005968 /*
5969 * Initialize the boot_pagesets that are going to be used
5970 * for bootstrapping processors. The real pagesets for
5971 * each zone will be allocated later when the per cpu
5972 * allocator is available.
5973 *
5974 * boot_pagesets are used also for bootstrapping offline
5975 * cpus if the system is already booted because the pagesets
5976 * are needed to initialize allocators on a specific cpu too.
5977 * F.e. the percpu allocator needs the page allocator which
5978 * needs the percpu allocator in order to allocate its pagesets
5979 * (a chicken-egg dilemma).
5980 */
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005981 for_each_possible_cpu(cpu)
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005982 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
5983
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005984 mminit_verify_zonelist();
5985 cpuset_init_current_mems_allowed();
5986}
5987
Haicheng Li4eaf3f62010-05-24 14:32:52 -07005988/*
Haicheng Li4eaf3f62010-05-24 14:32:52 -07005989 * unless system_state == SYSTEM_BOOTING.
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005990 *
Michal Hocko72675e12017-09-06 16:20:24 -07005991 * __ref due to call of __init annotated helper build_all_zonelists_init
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005992 * [protected by SYSTEM_BOOTING].
Haicheng Li4eaf3f62010-05-24 14:32:52 -07005993 */
Michal Hocko72675e12017-09-06 16:20:24 -07005994void __ref build_all_zonelists(pg_data_t *pgdat)
Yasunori Goto68113782006-06-23 02:03:11 -07005995{
David Hildenbrand0a18e602020-08-06 23:25:27 -07005996 unsigned long vm_total_pages;
5997
Yasunori Goto68113782006-06-23 02:03:11 -07005998 if (system_state == SYSTEM_BOOTING) {
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005999 build_all_zonelists_init();
Yasunori Goto68113782006-06-23 02:03:11 -07006000 } else {
Michal Hocko11cd8632017-09-06 16:20:34 -07006001 __build_all_zonelists(pgdat);
Yasunori Goto68113782006-06-23 02:03:11 -07006002 /* cpuset refresh routine should be here */
6003 }
David Hildenbrand56b94132020-08-06 23:25:30 -07006004 /* Get the number of free pages beyond high watermark in all zones. */
6005 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
Mel Gorman9ef9acb2007-10-16 01:25:54 -07006006 /*
6007 * Disable grouping by mobility if the number of pages in the
6008 * system is too low to allow the mechanism to work. It would be
6009 * more accurate, but expensive to check per-zone. This check is
6010 * made on memory-hotadd so a system can start with mobility
6011 * disabled and enable it later
6012 */
Mel Gormand9c23402007-10-16 01:26:01 -07006013 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
Mel Gorman9ef9acb2007-10-16 01:25:54 -07006014 page_group_by_mobility_disabled = 1;
6015 else
6016 page_group_by_mobility_disabled = 0;
6017
Alexey Dobriyance0725f2019-03-05 15:48:29 -08006018 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n",
Joe Perches756a0252016-03-17 14:19:47 -07006019 nr_online_nodes,
Joe Perches756a0252016-03-17 14:19:47 -07006020 page_group_by_mobility_disabled ? "off" : "on",
6021 vm_total_pages);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006022#ifdef CONFIG_NUMA
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006023 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006024#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07006025}
6026
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006027/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
6028static bool __meminit
6029overlap_memmap_init(unsigned long zone, unsigned long *pfn)
6030{
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006031 static struct memblock_region *r;
6032
6033 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
6034 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
Mike Rapoportcc6de162020-10-13 16:58:30 -07006035 for_each_mem_region(r) {
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006036 if (*pfn < memblock_region_memory_end_pfn(r))
6037 break;
6038 }
6039 }
6040 if (*pfn >= memblock_region_memory_base_pfn(r) &&
6041 memblock_is_mirror(r)) {
6042 *pfn = memblock_region_memory_end_pfn(r);
6043 return true;
6044 }
6045 }
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006046 return false;
6047}
6048
Linus Torvalds1da177e2005-04-16 15:20:36 -07006049/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07006050 * Initially all pages are reserved - free ones are freed
Mike Rapoportc6ffc5c2018-10-30 15:09:30 -07006051 * up by memblock_free_all() once the early boot process is
Linus Torvalds1da177e2005-04-16 15:20:36 -07006052 * done. Non-atomic initialization, single-pass.
David Hildenbrandd882c002020-10-15 20:08:19 -07006053 *
6054 * All aligned pageblocks are initialized to the specified migratetype
6055 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
6056 * zone stats (e.g., nr_isolate_pageblock) are touched.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006057 */
Matt Tolentinoc09b4242006-01-17 07:03:44 +01006058void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
Baoquan He98b57682020-12-29 15:14:37 -08006059 unsigned long start_pfn, unsigned long zone_end_pfn,
David Hildenbrandd882c002020-10-15 20:08:19 -07006060 enum meminit_context context,
6061 struct vmem_altmap *altmap, int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006062{
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006063 unsigned long pfn, end_pfn = start_pfn + size;
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006064 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006065
Hugh Dickins22b31ee2009-01-06 14:40:09 -08006066 if (highest_memmap_pfn < end_pfn - 1)
6067 highest_memmap_pfn = end_pfn - 1;
6068
Alexander Duyck966cf442018-10-26 15:07:52 -07006069#ifdef CONFIG_ZONE_DEVICE
Dan Williams4b94ffd2016-01-15 16:56:22 -08006070 /*
6071 * Honor reservation requested by the driver for this ZONE_DEVICE
Alexander Duyck966cf442018-10-26 15:07:52 -07006072 * memory. We limit the total number of pages to initialize to just
6073 * those that might contain the memory mapping. We will defer the
6074 * ZONE_DEVICE page initialization until after we have released
6075 * the hotplug lock.
Dan Williams4b94ffd2016-01-15 16:56:22 -08006076 */
Alexander Duyck966cf442018-10-26 15:07:52 -07006077 if (zone == ZONE_DEVICE) {
6078 if (!altmap)
6079 return;
6080
6081 if (start_pfn == altmap->base_pfn)
6082 start_pfn += altmap->reserve;
6083 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
6084 }
6085#endif
Dan Williams4b94ffd2016-01-15 16:56:22 -08006086
David Hildenbrand948c4362020-02-03 17:33:59 -08006087 for (pfn = start_pfn; pfn < end_pfn; ) {
Dave Hansena2f3aa022007-01-10 23:15:30 -08006088 /*
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07006089 * There can be holes in boot-time mem_map[]s handed to this
6090 * function. They do not exist on hotplugged memory.
Dave Hansena2f3aa022007-01-10 23:15:30 -08006091 */
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006092 if (context == MEMINIT_EARLY) {
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006093 if (overlap_memmap_init(zone, &pfn))
6094 continue;
Baoquan He98b57682020-12-29 15:14:37 -08006095 if (defer_init(nid, pfn, zone_end_pfn))
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006096 break;
Dave Hansena2f3aa022007-01-10 23:15:30 -08006097 }
Mel Gormanac5d2532015-06-30 14:57:20 -07006098
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006099 page = pfn_to_page(pfn);
6100 __init_single_page(page, pfn, zone, nid);
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006101 if (context == MEMINIT_HOTPLUG)
Alexander Duyckd483da52018-10-26 15:07:48 -07006102 __SetPageReserved(page);
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006103
Mel Gormanac5d2532015-06-30 14:57:20 -07006104 /*
David Hildenbrandd882c002020-10-15 20:08:19 -07006105 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
6106 * such that unmovable allocations won't be scattered all
6107 * over the place during system boot.
Mel Gormanac5d2532015-06-30 14:57:20 -07006108 */
David Hildenbrand4eb29bd2020-10-15 20:08:15 -07006109 if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
David Hildenbrandd882c002020-10-15 20:08:19 -07006110 set_pageblock_migratetype(page, migratetype);
Michal Hocko9b6e63c2017-10-03 16:16:19 -07006111 cond_resched();
Mel Gormanac5d2532015-06-30 14:57:20 -07006112 }
David Hildenbrand948c4362020-02-03 17:33:59 -08006113 pfn++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006114 }
6115}
6116
Alexander Duyck966cf442018-10-26 15:07:52 -07006117#ifdef CONFIG_ZONE_DEVICE
6118void __ref memmap_init_zone_device(struct zone *zone,
6119 unsigned long start_pfn,
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006120 unsigned long nr_pages,
Alexander Duyck966cf442018-10-26 15:07:52 -07006121 struct dev_pagemap *pgmap)
6122{
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006123 unsigned long pfn, end_pfn = start_pfn + nr_pages;
Alexander Duyck966cf442018-10-26 15:07:52 -07006124 struct pglist_data *pgdat = zone->zone_pgdat;
Christoph Hellwig514caf22019-06-26 14:27:13 +02006125 struct vmem_altmap *altmap = pgmap_altmap(pgmap);
Alexander Duyck966cf442018-10-26 15:07:52 -07006126 unsigned long zone_idx = zone_idx(zone);
6127 unsigned long start = jiffies;
6128 int nid = pgdat->node_id;
6129
Dan Williams46d945a2019-07-18 15:58:18 -07006130 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE))
Alexander Duyck966cf442018-10-26 15:07:52 -07006131 return;
6132
6133 /*
Mike Rapoportce6ee462021-06-28 19:33:26 -07006134 * The call to memmap_init should have already taken care
Alexander Duyck966cf442018-10-26 15:07:52 -07006135 * of the pages reserved for the memmap, so we can just jump to
6136 * the end of that region and start processing the device pages.
6137 */
Christoph Hellwig514caf22019-06-26 14:27:13 +02006138 if (altmap) {
Alexander Duyck966cf442018-10-26 15:07:52 -07006139 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006140 nr_pages = end_pfn - start_pfn;
Alexander Duyck966cf442018-10-26 15:07:52 -07006141 }
6142
6143 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
6144 struct page *page = pfn_to_page(pfn);
6145
6146 __init_single_page(page, pfn, zone_idx, nid);
6147
6148 /*
6149 * Mark page reserved as it will need to wait for onlining
6150 * phase for it to be fully associated with a zone.
6151 *
6152 * We can use the non-atomic __set_bit operation for setting
6153 * the flag as we are still initializing the pages.
6154 */
6155 __SetPageReserved(page);
6156
6157 /*
Christoph Hellwig8a164fe2019-06-26 14:27:21 +02006158 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
6159 * and zone_device_data. It is a bug if a ZONE_DEVICE page is
6160 * ever freed or placed on a driver-private list.
Alexander Duyck966cf442018-10-26 15:07:52 -07006161 */
6162 page->pgmap = pgmap;
Christoph Hellwig8a164fe2019-06-26 14:27:21 +02006163 page->zone_device_data = NULL;
Alexander Duyck966cf442018-10-26 15:07:52 -07006164
6165 /*
6166 * Mark the block movable so that blocks are reserved for
6167 * movable at startup. This will force kernel allocations
6168 * to reserve their blocks rather than leaking throughout
6169 * the address space during boot when many long-lived
6170 * kernel allocations are made.
6171 *
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006172 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
Dan Williamsba72b4c2019-07-18 15:58:26 -07006173 * because this is done early in section_activate()
Alexander Duyck966cf442018-10-26 15:07:52 -07006174 */
David Hildenbrand4eb29bd2020-10-15 20:08:15 -07006175 if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
Alexander Duyck966cf442018-10-26 15:07:52 -07006176 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
6177 cond_resched();
6178 }
6179 }
6180
Christoph Hellwigfdc029b2019-08-18 11:05:55 +02006181 pr_info("%s initialised %lu pages in %ums\n", __func__,
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006182 nr_pages, jiffies_to_msecs(jiffies - start));
Alexander Duyck966cf442018-10-26 15:07:52 -07006183}
6184
6185#endif
Andi Kleen1e548de2008-02-04 22:29:26 -08006186static void __meminit zone_init_free_lists(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006187{
Mel Gorman7aeb09f2014-06-04 16:10:21 -07006188 unsigned int order, t;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07006189 for_each_migratetype_order(order, t) {
6190 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006191 zone->free_area[order].nr_free = 0;
6192 }
6193}
6194
Mike Rapoport4c841912021-03-12 21:07:12 -08006195#if !defined(CONFIG_FLAT_NODE_MEM_MAP)
6196/*
6197 * Only struct pages that correspond to ranges defined by memblock.memory
6198 * are zeroed and initialized by going through __init_single_page() during
Mike Rapoportce6ee462021-06-28 19:33:26 -07006199 * memmap_init_zone_range().
Mike Rapoport4c841912021-03-12 21:07:12 -08006200 *
6201 * But, there could be struct pages that correspond to holes in
6202 * memblock.memory. This can happen because of the following reasons:
6203 * - physical memory bank size is not necessarily the exact multiple of the
6204 * arbitrary section size
6205 * - early reserved memory may not be listed in memblock.memory
6206 * - memory layouts defined with memmap= kernel parameter may not align
6207 * nicely with memmap sections
6208 *
6209 * Explicitly initialize those struct pages so that:
6210 * - PG_Reserved is set
6211 * - zone and node links point to zone and node that span the page if the
6212 * hole is in the middle of a zone
6213 * - zone and node links point to adjacent zone/node if the hole falls on
6214 * the zone boundary; the pages in such holes will be prepended to the
6215 * zone/node above the hole except for the trailing pages in the last
6216 * section that will be appended to the zone/node below.
6217 */
Mike Rapoportce6ee462021-06-28 19:33:26 -07006218static void __init init_unavailable_range(unsigned long spfn,
6219 unsigned long epfn,
6220 int zone, int node)
Mike Rapoport4c841912021-03-12 21:07:12 -08006221{
6222 unsigned long pfn;
6223 u64 pgcnt = 0;
6224
6225 for (pfn = spfn; pfn < epfn; pfn++) {
6226 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
6227 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
6228 + pageblock_nr_pages - 1;
6229 continue;
6230 }
6231 __init_single_page(pfn_to_page(pfn), pfn, zone, node);
6232 __SetPageReserved(pfn_to_page(pfn));
6233 pgcnt++;
6234 }
6235
Mike Rapoportce6ee462021-06-28 19:33:26 -07006236 if (pgcnt)
6237 pr_info("On node %d, zone %s: %lld pages in unavailable ranges",
6238 node, zone_names[zone], pgcnt);
Mike Rapoport4c841912021-03-12 21:07:12 -08006239}
6240#else
Mike Rapoportce6ee462021-06-28 19:33:26 -07006241static inline void init_unavailable_range(unsigned long spfn,
6242 unsigned long epfn,
6243 int zone, int node)
Mike Rapoport4c841912021-03-12 21:07:12 -08006244{
Mike Rapoport4c841912021-03-12 21:07:12 -08006245}
6246#endif
6247
Mike Rapoportce6ee462021-06-28 19:33:26 -07006248static void __init memmap_init_zone_range(struct zone *zone,
6249 unsigned long start_pfn,
6250 unsigned long end_pfn,
6251 unsigned long *hole_pfn)
Pavel Tatashindfb3ccd2018-10-26 15:09:32 -07006252{
Mike Rapoportce6ee462021-06-28 19:33:26 -07006253 unsigned long zone_start_pfn = zone->zone_start_pfn;
6254 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
6255 int nid = zone_to_nid(zone), zone_id = zone_idx(zone);
6256
6257 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);
6258 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);
6259
6260 if (start_pfn >= end_pfn)
6261 return;
6262
6263 memmap_init_zone(end_pfn - start_pfn, nid, zone_id, start_pfn,
6264 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
6265
6266 if (*hole_pfn < start_pfn)
6267 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
6268
6269 *hole_pfn = end_pfn;
6270}
6271
6272void __init __weak memmap_init(void)
6273{
Baoquan He73a6e472020-06-03 15:57:55 -07006274 unsigned long start_pfn, end_pfn;
Mike Rapoportce6ee462021-06-28 19:33:26 -07006275 unsigned long hole_pfn = 0;
6276 int i, j, zone_id, nid;
Baoquan He73a6e472020-06-03 15:57:55 -07006277
Mike Rapoportce6ee462021-06-28 19:33:26 -07006278 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
6279 struct pglist_data *node = NODE_DATA(nid);
Baoquan He73a6e472020-06-03 15:57:55 -07006280
Mike Rapoportce6ee462021-06-28 19:33:26 -07006281 for (j = 0; j < MAX_NR_ZONES; j++) {
6282 struct zone *zone = node->node_zones + j;
6283
6284 if (!populated_zone(zone))
6285 continue;
6286
6287 memmap_init_zone_range(zone, start_pfn, end_pfn,
6288 &hole_pfn);
6289 zone_id = j;
Baoquan He73a6e472020-06-03 15:57:55 -07006290 }
6291 }
Mike Rapoport4c841912021-03-12 21:07:12 -08006292
6293#ifdef CONFIG_SPARSEMEM
6294 /*
Mike Rapoportce6ee462021-06-28 19:33:26 -07006295 * Initialize the memory map for hole in the range [memory_end,
6296 * section_end].
6297 * Append the pages in this hole to the highest zone in the last
6298 * node.
6299 * The call to init_unavailable_range() is outside the ifdef to
6300 * silence the compiler warining about zone_id set but not used;
6301 * for FLATMEM it is a nop anyway
Mike Rapoport4c841912021-03-12 21:07:12 -08006302 */
Mike Rapoportce6ee462021-06-28 19:33:26 -07006303 end_pfn = round_up(end_pfn, PAGES_PER_SECTION);
Mike Rapoport4c841912021-03-12 21:07:12 -08006304 if (hole_pfn < end_pfn)
Mike Rapoport4c841912021-03-12 21:07:12 -08006305#endif
Mike Rapoportce6ee462021-06-28 19:33:26 -07006306 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);
6307}
Mike Rapoport4c841912021-03-12 21:07:12 -08006308
Mike Rapoportce6ee462021-06-28 19:33:26 -07006309/* A stub for backwards compatibility with custom implementatin on IA-64 */
6310void __meminit __weak arch_memmap_init(unsigned long size, int nid,
6311 unsigned long zone,
6312 unsigned long range_start_pfn)
6313{
Pavel Tatashindfb3ccd2018-10-26 15:09:32 -07006314}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006315
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006316static int zone_batchsize(struct zone *zone)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006317{
David Howells3a6be872009-05-06 16:03:03 -07006318#ifdef CONFIG_MMU
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006319 int batch;
6320
6321 /*
6322 * The per-cpu-pages pools are set to around 1000th of the
Aaron Lud8a759b2018-08-17 15:49:14 -07006323 * size of the zone.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006324 */
Arun KS9705bea2018-12-28 00:34:24 -08006325 batch = zone_managed_pages(zone) / 1024;
Aaron Lud8a759b2018-08-17 15:49:14 -07006326 /* But no more than a meg. */
6327 if (batch * PAGE_SIZE > 1024 * 1024)
6328 batch = (1024 * 1024) / PAGE_SIZE;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006329 batch /= 4; /* We effectively *= 4 below */
6330 if (batch < 1)
6331 batch = 1;
6332
6333 /*
Nick Piggin0ceaacc2005-12-04 13:55:25 +11006334 * Clamp the batch to a 2^n - 1 value. Having a power
6335 * of 2 value was found to be more likely to have
6336 * suboptimal cache aliasing properties in some cases.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006337 *
Nick Piggin0ceaacc2005-12-04 13:55:25 +11006338 * For example if 2 tasks are alternately allocating
6339 * batches of pages, one task can end up with a lot
6340 * of pages of one half of the possible page colors
6341 * and the other with pages of the other colors.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006342 */
David Howells91552032009-05-06 16:03:02 -07006343 batch = rounddown_pow_of_two(batch + batch/2) - 1;
Seth, Rohitba56e912005-10-29 18:15:47 -07006344
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006345 return batch;
David Howells3a6be872009-05-06 16:03:03 -07006346
6347#else
6348 /* The deferral and batching of frees should be suppressed under NOMMU
6349 * conditions.
6350 *
6351 * The problem is that NOMMU needs to be able to allocate large chunks
6352 * of contiguous memory as there's no hardware page translation to
6353 * assemble apparent contiguous memory from discontiguous pages.
6354 *
6355 * Queueing large contiguous runs of pages for batching, however,
6356 * causes the pages to actually be freed in smaller chunks. As there
6357 * can be a significant delay between the individual batches being
6358 * recycled, this leads to the once large chunks of space being
6359 * fragmented and becoming unavailable for high-order allocations.
6360 */
6361 return 0;
6362#endif
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006363}
6364
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006365/*
6366 * pcp->high and pcp->batch values are related and dependent on one another:
6367 * ->batch must never be higher then ->high.
6368 * The following function updates them in a safe manner without read side
6369 * locking.
6370 *
6371 * Any new users of pcp->batch and pcp->high should ensure they can cope with
Randy Dunlap047b9962020-08-11 18:33:14 -07006372 * those fields changing asynchronously (acording to the above rule).
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006373 *
6374 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
6375 * outside of boot time (or some other assurance that no concurrent updaters
6376 * exist).
6377 */
6378static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
6379 unsigned long batch)
6380{
6381 /* start with a fail safe value for batch */
6382 pcp->batch = 1;
6383 smp_wmb();
6384
6385 /* Update high, then batch, in order */
6386 pcp->high = high;
6387 smp_wmb();
6388
6389 pcp->batch = batch;
6390}
6391
Cody P Schafer36640332013-07-03 15:01:40 -07006392/* a companion to pageset_set_high() */
Cody P Schafer4008bab2013-07-03 15:01:28 -07006393static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
6394{
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006395 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
Cody P Schafer4008bab2013-07-03 15:01:28 -07006396}
6397
Cody P Schafer88c90db2013-07-03 15:01:35 -07006398static void pageset_init(struct per_cpu_pageset *p)
Christoph Lameter2caaad42005-06-21 17:15:00 -07006399{
6400 struct per_cpu_pages *pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07006401 int migratetype;
Christoph Lameter2caaad42005-06-21 17:15:00 -07006402
Magnus Damm1c6fe942005-10-26 01:58:59 -07006403 memset(p, 0, sizeof(*p));
6404
Christoph Lameter3dfa5722008-02-04 22:29:19 -08006405 pcp = &p->pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07006406 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
6407 INIT_LIST_HEAD(&pcp->lists[migratetype]);
Christoph Lameter2caaad42005-06-21 17:15:00 -07006408}
6409
Cody P Schafer88c90db2013-07-03 15:01:35 -07006410static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
6411{
6412 pageset_init(p);
6413 pageset_set_batch(p, batch);
6414}
6415
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006416/*
Cody P Schafer36640332013-07-03 15:01:40 -07006417 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006418 * to the value high for the pageset p.
6419 */
Cody P Schafer36640332013-07-03 15:01:40 -07006420static void pageset_set_high(struct per_cpu_pageset *p,
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006421 unsigned long high)
6422{
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006423 unsigned long batch = max(1UL, high / 4);
6424 if ((high / 4) > (PAGE_SHIFT * 8))
6425 batch = PAGE_SHIFT * 8;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006426
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006427 pageset_update(&p->pcp, high, batch);
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006428}
6429
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006430static void pageset_set_high_and_batch(struct zone *zone,
6431 struct per_cpu_pageset *pcp)
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006432{
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006433 if (percpu_pagelist_fraction)
Cody P Schafer36640332013-07-03 15:01:40 -07006434 pageset_set_high(pcp,
Arun KS9705bea2018-12-28 00:34:24 -08006435 (zone_managed_pages(zone) /
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006436 percpu_pagelist_fraction));
6437 else
6438 pageset_set_batch(pcp, zone_batchsize(zone));
6439}
6440
Cody P Schafer169f6c12013-07-03 15:01:41 -07006441static void __meminit zone_pageset_init(struct zone *zone, int cpu)
6442{
6443 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
6444
6445 pageset_init(pcp);
6446 pageset_set_high_and_batch(zone, pcp);
6447}
6448
Michal Hocko72675e12017-09-06 16:20:24 -07006449void __meminit setup_zone_pageset(struct zone *zone)
Wu Fengguang319774e2010-05-24 14:32:49 -07006450{
6451 int cpu;
Wu Fengguang319774e2010-05-24 14:32:49 -07006452 zone->pageset = alloc_percpu(struct per_cpu_pageset);
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006453 for_each_possible_cpu(cpu)
6454 zone_pageset_init(zone, cpu);
Wu Fengguang319774e2010-05-24 14:32:49 -07006455}
6456
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006457/*
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006458 * Allocate per cpu pagesets and initialize them.
6459 * Before this call only boot pagesets were available.
Christoph Lameter2caaad42005-06-21 17:15:00 -07006460 */
Al Viro78d99552005-12-15 09:18:25 +00006461void __init setup_per_cpu_pageset(void)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006462{
Mel Gormanb4911ea2016-08-04 15:31:49 -07006463 struct pglist_data *pgdat;
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006464 struct zone *zone;
Sandipan Dasb418a0f2020-06-03 15:59:11 -07006465 int __maybe_unused cpu;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006466
Wu Fengguang319774e2010-05-24 14:32:49 -07006467 for_each_populated_zone(zone)
6468 setup_zone_pageset(zone);
Mel Gormanb4911ea2016-08-04 15:31:49 -07006469
Sandipan Dasb418a0f2020-06-03 15:59:11 -07006470#ifdef CONFIG_NUMA
6471 /*
6472 * Unpopulated zones continue using the boot pagesets.
6473 * The numa stats for these pagesets need to be reset.
6474 * Otherwise, they will end up skewing the stats of
6475 * the nodes these zones are associated with.
6476 */
6477 for_each_possible_cpu(cpu) {
6478 struct per_cpu_pageset *pcp = &per_cpu(boot_pageset, cpu);
6479 memset(pcp->vm_numa_stat_diff, 0,
6480 sizeof(pcp->vm_numa_stat_diff));
6481 }
6482#endif
6483
Mel Gormanb4911ea2016-08-04 15:31:49 -07006484 for_each_online_pgdat(pgdat)
6485 pgdat->per_cpu_nodestats =
6486 alloc_percpu(struct per_cpu_nodestat);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006487}
6488
Matt Tolentinoc09b4242006-01-17 07:03:44 +01006489static __meminit void zone_pcp_init(struct zone *zone)
Dave Hansened8ece22005-10-29 18:16:50 -07006490{
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006491 /*
6492 * per cpu subsystem is not up at this point. The following code
6493 * relies on the ability of the linker to provide the
6494 * offset of a (static) per cpu variable into the per cpu area.
6495 */
6496 zone->pageset = &boot_pageset;
Dave Hansened8ece22005-10-29 18:16:50 -07006497
Xishi Qiub38a8722013-11-12 15:07:20 -08006498 if (populated_zone(zone))
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006499 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
6500 zone->name, zone->present_pages,
6501 zone_batchsize(zone));
Dave Hansened8ece22005-10-29 18:16:50 -07006502}
6503
Michal Hockodc0bbf32017-07-06 15:37:35 -07006504void __meminit init_currently_empty_zone(struct zone *zone,
Yasunori Goto718127c2006-06-23 02:03:10 -07006505 unsigned long zone_start_pfn,
Yaowei Baib171e402015-11-05 18:47:06 -08006506 unsigned long size)
Dave Hansened8ece22005-10-29 18:16:50 -07006507{
6508 struct pglist_data *pgdat = zone->zone_pgdat;
Wei Yang8f416832018-11-30 14:09:07 -08006509 int zone_idx = zone_idx(zone) + 1;
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07006510
Wei Yang8f416832018-11-30 14:09:07 -08006511 if (zone_idx > pgdat->nr_zones)
6512 pgdat->nr_zones = zone_idx;
Dave Hansened8ece22005-10-29 18:16:50 -07006513
Dave Hansened8ece22005-10-29 18:16:50 -07006514 zone->zone_start_pfn = zone_start_pfn;
6515
Mel Gorman708614e2008-07-23 21:26:51 -07006516 mminit_dprintk(MMINIT_TRACE, "memmap_init",
6517 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
6518 pgdat->node_id,
6519 (unsigned long)zone_idx(zone),
6520 zone_start_pfn, (zone_start_pfn + size));
6521
Andi Kleen1e548de2008-02-04 22:29:26 -08006522 zone_init_free_lists(zone);
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07006523 zone->initialized = 1;
Dave Hansened8ece22005-10-29 18:16:50 -07006524}
6525
Mel Gormanc7132162006-09-27 01:49:43 -07006526/**
Mel Gormanc7132162006-09-27 01:49:43 -07006527 * get_pfn_range_for_nid - Return the start and end page frames for a node
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006528 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
6529 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
6530 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
Mel Gormanc7132162006-09-27 01:49:43 -07006531 *
6532 * It returns the start and end page frame of a node based on information
Zhang Zhen7d018172014-06-04 16:10:53 -07006533 * provided by memblock_set_node(). If called for a node
Mel Gormanc7132162006-09-27 01:49:43 -07006534 * with no available memory, a warning is printed and the start and end
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006535 * PFNs will be 0.
Mel Gormanc7132162006-09-27 01:49:43 -07006536 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006537void __init get_pfn_range_for_nid(unsigned int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006538 unsigned long *start_pfn, unsigned long *end_pfn)
6539{
Tejun Heoc13291a2011-07-12 10:46:30 +02006540 unsigned long this_start_pfn, this_end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006541 int i;
Tejun Heoc13291a2011-07-12 10:46:30 +02006542
Mel Gormanc7132162006-09-27 01:49:43 -07006543 *start_pfn = -1UL;
6544 *end_pfn = 0;
6545
Tejun Heoc13291a2011-07-12 10:46:30 +02006546 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
6547 *start_pfn = min(*start_pfn, this_start_pfn);
6548 *end_pfn = max(*end_pfn, this_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006549 }
6550
Christoph Lameter633c0662007-10-16 01:25:37 -07006551 if (*start_pfn == -1UL)
Mel Gormanc7132162006-09-27 01:49:43 -07006552 *start_pfn = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07006553}
6554
6555/*
Mel Gorman2a1e2742007-07-17 04:03:12 -07006556 * This finds a zone that can be used for ZONE_MOVABLE pages. The
6557 * assumption is made that zones within a node are ordered in monotonic
6558 * increasing memory addresses so that the "highest" populated zone is used
6559 */
Adrian Bunkb69a7282008-07-23 21:28:12 -07006560static void __init find_usable_zone_for_movable(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07006561{
6562 int zone_index;
6563 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
6564 if (zone_index == ZONE_MOVABLE)
6565 continue;
6566
6567 if (arch_zone_highest_possible_pfn[zone_index] >
6568 arch_zone_lowest_possible_pfn[zone_index])
6569 break;
6570 }
6571
6572 VM_BUG_ON(zone_index == -1);
6573 movable_zone = zone_index;
6574}
6575
6576/*
6577 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006578 * because it is sized independent of architecture. Unlike the other zones,
Mel Gorman2a1e2742007-07-17 04:03:12 -07006579 * the starting point for ZONE_MOVABLE is not fixed. It may be different
6580 * in each node depending on the size of each node and how evenly kernelcore
6581 * is distributed. This helper function adjusts the zone ranges
6582 * provided by the architecture for a given node by using the end of the
6583 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
6584 * zones within a node are in order of monotonic increases memory addresses
6585 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006586static void __init adjust_zone_range_for_zone_movable(int nid,
Mel Gorman2a1e2742007-07-17 04:03:12 -07006587 unsigned long zone_type,
6588 unsigned long node_start_pfn,
6589 unsigned long node_end_pfn,
6590 unsigned long *zone_start_pfn,
6591 unsigned long *zone_end_pfn)
6592{
6593 /* Only adjust if ZONE_MOVABLE is on this node */
6594 if (zone_movable_pfn[nid]) {
6595 /* Size ZONE_MOVABLE */
6596 if (zone_type == ZONE_MOVABLE) {
6597 *zone_start_pfn = zone_movable_pfn[nid];
6598 *zone_end_pfn = min(node_end_pfn,
6599 arch_zone_highest_possible_pfn[movable_zone]);
6600
Xishi Qiue506b992016-10-07 16:58:06 -07006601 /* Adjust for ZONE_MOVABLE starting within this range */
6602 } else if (!mirrored_kernelcore &&
6603 *zone_start_pfn < zone_movable_pfn[nid] &&
6604 *zone_end_pfn > zone_movable_pfn[nid]) {
6605 *zone_end_pfn = zone_movable_pfn[nid];
6606
Mel Gorman2a1e2742007-07-17 04:03:12 -07006607 /* Check if this whole range is within ZONE_MOVABLE */
6608 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
6609 *zone_start_pfn = *zone_end_pfn;
6610 }
6611}
6612
6613/*
Mel Gormanc7132162006-09-27 01:49:43 -07006614 * Return the number of pages a zone spans in a node, including holes
6615 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
6616 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006617static unsigned long __init zone_spanned_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006618 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006619 unsigned long node_start_pfn,
6620 unsigned long node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07006621 unsigned long *zone_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07006622 unsigned long *zone_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006623{
Linxu Fang299c83d2019-05-13 17:19:17 -07006624 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6625 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Xishi Qiub5685e92015-09-08 15:04:16 -07006626 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07006627 if (!node_start_pfn && !node_end_pfn)
6628 return 0;
6629
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006630 /* Get the start and end of the zone */
Linxu Fang299c83d2019-05-13 17:19:17 -07006631 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6632 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman2a1e2742007-07-17 04:03:12 -07006633 adjust_zone_range_for_zone_movable(nid, zone_type,
6634 node_start_pfn, node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07006635 zone_start_pfn, zone_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006636
6637 /* Check that this node has pages within the zone's required range */
Taku Izumid91749c2016-03-15 14:55:18 -07006638 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006639 return 0;
6640
6641 /* Move the zone boundaries inside the node if necessary */
Taku Izumid91749c2016-03-15 14:55:18 -07006642 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
6643 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006644
6645 /* Return the spanned pages */
Taku Izumid91749c2016-03-15 14:55:18 -07006646 return *zone_end_pfn - *zone_start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006647}
6648
6649/*
6650 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006651 * then all holes in the requested range will be accounted for.
Mel Gormanc7132162006-09-27 01:49:43 -07006652 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006653unsigned long __init __absent_pages_in_range(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006654 unsigned long range_start_pfn,
6655 unsigned long range_end_pfn)
6656{
Tejun Heo96e907d2011-07-12 10:46:29 +02006657 unsigned long nr_absent = range_end_pfn - range_start_pfn;
6658 unsigned long start_pfn, end_pfn;
6659 int i;
Mel Gormanc7132162006-09-27 01:49:43 -07006660
Tejun Heo96e907d2011-07-12 10:46:29 +02006661 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
6662 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
6663 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
6664 nr_absent -= end_pfn - start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006665 }
Tejun Heo96e907d2011-07-12 10:46:29 +02006666 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07006667}
6668
6669/**
6670 * absent_pages_in_range - Return number of page frames in holes within a range
6671 * @start_pfn: The start PFN to start searching for holes
6672 * @end_pfn: The end PFN to stop searching for holes
6673 *
Mike Rapoporta862f682019-03-05 15:48:42 -08006674 * Return: the number of pages frames in memory holes within a range.
Mel Gormanc7132162006-09-27 01:49:43 -07006675 */
6676unsigned long __init absent_pages_in_range(unsigned long start_pfn,
6677 unsigned long end_pfn)
6678{
6679 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
6680}
6681
6682/* Return the number of page frames in holes in a zone on a node */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006683static unsigned long __init zone_absent_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006684 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006685 unsigned long node_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07006686 unsigned long node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006687{
Tejun Heo96e907d2011-07-12 10:46:29 +02006688 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6689 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Mel Gorman9c7cd682006-09-27 01:49:58 -07006690 unsigned long zone_start_pfn, zone_end_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07006691 unsigned long nr_absent;
Mel Gorman9c7cd682006-09-27 01:49:58 -07006692
Xishi Qiub5685e92015-09-08 15:04:16 -07006693 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07006694 if (!node_start_pfn && !node_end_pfn)
6695 return 0;
6696
Tejun Heo96e907d2011-07-12 10:46:29 +02006697 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6698 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman9c7cd682006-09-27 01:49:58 -07006699
Mel Gorman2a1e2742007-07-17 04:03:12 -07006700 adjust_zone_range_for_zone_movable(nid, zone_type,
6701 node_start_pfn, node_end_pfn,
6702 &zone_start_pfn, &zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07006703 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
6704
6705 /*
6706 * ZONE_MOVABLE handling.
6707 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
6708 * and vice versa.
6709 */
Xishi Qiue506b992016-10-07 16:58:06 -07006710 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
6711 unsigned long start_pfn, end_pfn;
6712 struct memblock_region *r;
Taku Izumi342332e2016-03-15 14:55:22 -07006713
Mike Rapoportcc6de162020-10-13 16:58:30 -07006714 for_each_mem_region(r) {
Xishi Qiue506b992016-10-07 16:58:06 -07006715 start_pfn = clamp(memblock_region_memory_base_pfn(r),
6716 zone_start_pfn, zone_end_pfn);
6717 end_pfn = clamp(memblock_region_memory_end_pfn(r),
6718 zone_start_pfn, zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07006719
Xishi Qiue506b992016-10-07 16:58:06 -07006720 if (zone_type == ZONE_MOVABLE &&
6721 memblock_is_mirror(r))
6722 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07006723
Xishi Qiue506b992016-10-07 16:58:06 -07006724 if (zone_type == ZONE_NORMAL &&
6725 !memblock_is_mirror(r))
6726 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07006727 }
6728 }
6729
6730 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07006731}
Mel Gorman0e0b8642006-09-27 01:49:56 -07006732
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006733static void __init calculate_node_totalpages(struct pglist_data *pgdat,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006734 unsigned long node_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07006735 unsigned long node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006736{
Gu Zhengfebd5942015-06-24 16:57:02 -07006737 unsigned long realtotalpages = 0, totalpages = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07006738 enum zone_type i;
6739
Gu Zhengfebd5942015-06-24 16:57:02 -07006740 for (i = 0; i < MAX_NR_ZONES; i++) {
6741 struct zone *zone = pgdat->node_zones + i;
Taku Izumid91749c2016-03-15 14:55:18 -07006742 unsigned long zone_start_pfn, zone_end_pfn;
Mike Rapoport3f08a302020-06-03 15:57:02 -07006743 unsigned long spanned, absent;
Gu Zhengfebd5942015-06-24 16:57:02 -07006744 unsigned long size, real_size;
Mel Gormanc7132162006-09-27 01:49:43 -07006745
Mike Rapoport854e8842020-06-03 15:58:13 -07006746 spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
6747 node_start_pfn,
6748 node_end_pfn,
6749 &zone_start_pfn,
6750 &zone_end_pfn);
6751 absent = zone_absent_pages_in_node(pgdat->node_id, i,
6752 node_start_pfn,
6753 node_end_pfn);
Mike Rapoport3f08a302020-06-03 15:57:02 -07006754
6755 size = spanned;
6756 real_size = size - absent;
6757
Taku Izumid91749c2016-03-15 14:55:18 -07006758 if (size)
6759 zone->zone_start_pfn = zone_start_pfn;
6760 else
6761 zone->zone_start_pfn = 0;
Gu Zhengfebd5942015-06-24 16:57:02 -07006762 zone->spanned_pages = size;
6763 zone->present_pages = real_size;
6764
6765 totalpages += size;
6766 realtotalpages += real_size;
6767 }
6768
6769 pgdat->node_spanned_pages = totalpages;
Mel Gormanc7132162006-09-27 01:49:43 -07006770 pgdat->node_present_pages = realtotalpages;
6771 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
6772 realtotalpages);
6773}
6774
Mel Gorman835c1342007-10-16 01:25:47 -07006775#ifndef CONFIG_SPARSEMEM
6776/*
6777 * Calculate the size of the zone->blockflags rounded to an unsigned long
Mel Gormand9c23402007-10-16 01:26:01 -07006778 * Start by making sure zonesize is a multiple of pageblock_order by rounding
6779 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
Mel Gorman835c1342007-10-16 01:25:47 -07006780 * round what is now in bits to nearest long in bits, then return it in
6781 * bytes.
6782 */
Linus Torvalds7c455122013-02-18 09:58:02 -08006783static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
Mel Gorman835c1342007-10-16 01:25:47 -07006784{
6785 unsigned long usemapsize;
6786
Linus Torvalds7c455122013-02-18 09:58:02 -08006787 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
Mel Gormand9c23402007-10-16 01:26:01 -07006788 usemapsize = roundup(zonesize, pageblock_nr_pages);
6789 usemapsize = usemapsize >> pageblock_order;
Mel Gorman835c1342007-10-16 01:25:47 -07006790 usemapsize *= NR_PAGEBLOCK_BITS;
6791 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
6792
6793 return usemapsize / 8;
6794}
6795
Pavel Tatashin7cc2a952018-08-21 21:53:36 -07006796static void __ref setup_usemap(struct pglist_data *pgdat,
Linus Torvalds7c455122013-02-18 09:58:02 -08006797 struct zone *zone,
6798 unsigned long zone_start_pfn,
6799 unsigned long zonesize)
Mel Gorman835c1342007-10-16 01:25:47 -07006800{
Linus Torvalds7c455122013-02-18 09:58:02 -08006801 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
Mel Gorman835c1342007-10-16 01:25:47 -07006802 zone->pageblock_flags = NULL;
Mike Rapoport23a70522019-03-05 15:46:43 -08006803 if (usemapsize) {
Santosh Shilimkar67828322014-01-21 15:50:25 -08006804 zone->pageblock_flags =
Mike Rapoport26fb3da2019-03-11 23:30:42 -07006805 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
6806 pgdat->node_id);
Mike Rapoport23a70522019-03-05 15:46:43 -08006807 if (!zone->pageblock_flags)
6808 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
6809 usemapsize, zone->name, pgdat->node_id);
6810 }
Mel Gorman835c1342007-10-16 01:25:47 -07006811}
6812#else
Linus Torvalds7c455122013-02-18 09:58:02 -08006813static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
6814 unsigned long zone_start_pfn, unsigned long zonesize) {}
Mel Gorman835c1342007-10-16 01:25:47 -07006815#endif /* CONFIG_SPARSEMEM */
6816
Mel Gormand9c23402007-10-16 01:26:01 -07006817#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Mel Gormanba72cb82007-11-28 16:21:13 -08006818
Mel Gormand9c23402007-10-16 01:26:01 -07006819/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
Oscar Salvador03e85f92018-08-21 21:53:43 -07006820void __init set_pageblock_order(void)
Mel Gormand9c23402007-10-16 01:26:01 -07006821{
Andrew Morton955c1cd2012-05-29 15:06:31 -07006822 unsigned int order;
6823
Mel Gormand9c23402007-10-16 01:26:01 -07006824 /* Check that pageblock_nr_pages has not already been setup */
6825 if (pageblock_order)
6826 return;
6827
Andrew Morton955c1cd2012-05-29 15:06:31 -07006828 if (HPAGE_SHIFT > PAGE_SHIFT)
6829 order = HUGETLB_PAGE_ORDER;
6830 else
6831 order = MAX_ORDER - 1;
6832
Mel Gormand9c23402007-10-16 01:26:01 -07006833 /*
6834 * Assume the largest contiguous order of interest is a huge page.
Andrew Morton955c1cd2012-05-29 15:06:31 -07006835 * This value may be variable depending on boot parameters on IA64 and
6836 * powerpc.
Mel Gormand9c23402007-10-16 01:26:01 -07006837 */
6838 pageblock_order = order;
6839}
6840#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6841
Mel Gormanba72cb82007-11-28 16:21:13 -08006842/*
6843 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
Andrew Morton955c1cd2012-05-29 15:06:31 -07006844 * is unused as pageblock_order is set at compile-time. See
6845 * include/linux/pageblock-flags.h for the values of pageblock_order based on
6846 * the kernel config
Mel Gormanba72cb82007-11-28 16:21:13 -08006847 */
Oscar Salvador03e85f92018-08-21 21:53:43 -07006848void __init set_pageblock_order(void)
Mel Gormanba72cb82007-11-28 16:21:13 -08006849{
Mel Gormanba72cb82007-11-28 16:21:13 -08006850}
Mel Gormand9c23402007-10-16 01:26:01 -07006851
6852#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6853
Oscar Salvador03e85f92018-08-21 21:53:43 -07006854static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
Pavel Tatashin7cc2a952018-08-21 21:53:36 -07006855 unsigned long present_pages)
Jiang Liu01cefae2012-12-12 13:52:19 -08006856{
6857 unsigned long pages = spanned_pages;
6858
6859 /*
6860 * Provide a more accurate estimation if there are holes within
6861 * the zone and SPARSEMEM is in use. If there are holes within the
6862 * zone, each populated memory region may cost us one or two extra
6863 * memmap pages due to alignment because memmap pages for each
Masahiro Yamada89d790a2017-02-27 14:29:01 -08006864 * populated regions may not be naturally aligned on page boundary.
Jiang Liu01cefae2012-12-12 13:52:19 -08006865 * So the (present_pages >> 4) heuristic is a tradeoff for that.
6866 */
6867 if (spanned_pages > present_pages + (present_pages >> 4) &&
6868 IS_ENABLED(CONFIG_SPARSEMEM))
6869 pages = present_pages;
6870
6871 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
6872}
6873
Oscar Salvadorace1db32018-08-21 21:53:29 -07006874#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6875static void pgdat_init_split_queue(struct pglist_data *pgdat)
6876{
Yang Shi364c1ee2019-09-23 15:38:06 -07006877 struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
6878
6879 spin_lock_init(&ds_queue->split_queue_lock);
6880 INIT_LIST_HEAD(&ds_queue->split_queue);
6881 ds_queue->split_queue_len = 0;
Oscar Salvadorace1db32018-08-21 21:53:29 -07006882}
6883#else
6884static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
6885#endif
6886
6887#ifdef CONFIG_COMPACTION
6888static void pgdat_init_kcompactd(struct pglist_data *pgdat)
6889{
6890 init_waitqueue_head(&pgdat->kcompactd_wait);
6891}
6892#else
6893static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
6894#endif
6895
Oscar Salvador03e85f92018-08-21 21:53:43 -07006896static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006897{
Dave Hansen208d54e2005-10-29 18:16:52 -07006898 pgdat_resize_init(pgdat);
Oscar Salvadorace1db32018-08-21 21:53:29 -07006899
Oscar Salvadorace1db32018-08-21 21:53:29 -07006900 pgdat_init_split_queue(pgdat);
6901 pgdat_init_kcompactd(pgdat);
6902
Linus Torvalds1da177e2005-04-16 15:20:36 -07006903 init_waitqueue_head(&pgdat->kswapd_wait);
Mel Gorman55150612012-07-31 16:44:35 -07006904 init_waitqueue_head(&pgdat->pfmemalloc_wait);
Oscar Salvadorace1db32018-08-21 21:53:29 -07006905
Joonsoo Kimeefa864b2014-12-12 16:55:46 -08006906 pgdat_page_ext_init(pgdat);
Mel Gormana52633d2016-07-28 15:45:28 -07006907 spin_lock_init(&pgdat->lru_lock);
Johannes Weiner867e5e12019-11-30 17:55:34 -08006908 lruvec_init(&pgdat->__lruvec);
Oscar Salvador03e85f92018-08-21 21:53:43 -07006909}
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01006910
Oscar Salvador03e85f92018-08-21 21:53:43 -07006911static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
6912 unsigned long remaining_pages)
6913{
Arun KS9705bea2018-12-28 00:34:24 -08006914 atomic_long_set(&zone->managed_pages, remaining_pages);
Oscar Salvador03e85f92018-08-21 21:53:43 -07006915 zone_set_nid(zone, nid);
6916 zone->name = zone_names[idx];
6917 zone->zone_pgdat = NODE_DATA(nid);
6918 spin_lock_init(&zone->lock);
6919 zone_seqlock_init(zone);
6920 zone_pcp_init(zone);
6921}
6922
6923/*
6924 * Set up the zone data structures
6925 * - init pgdat internals
6926 * - init all zones belonging to this node
6927 *
6928 * NOTE: this function is only called during memory hotplug
6929 */
6930#ifdef CONFIG_MEMORY_HOTPLUG
6931void __ref free_area_init_core_hotplug(int nid)
6932{
6933 enum zone_type z;
6934 pg_data_t *pgdat = NODE_DATA(nid);
6935
6936 pgdat_init_internals(pgdat);
6937 for (z = 0; z < MAX_NR_ZONES; z++)
6938 zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
6939}
6940#endif
6941
6942/*
6943 * Set up the zone data structures:
6944 * - mark all pages reserved
6945 * - mark all memory queues empty
6946 * - clear the memory bitmaps
6947 *
6948 * NOTE: pgdat should get zeroed by caller.
6949 * NOTE: this function is only called during early init.
6950 */
6951static void __init free_area_init_core(struct pglist_data *pgdat)
6952{
6953 enum zone_type j;
6954 int nid = pgdat->node_id;
6955
6956 pgdat_init_internals(pgdat);
Johannes Weiner385386c2017-07-06 15:40:43 -07006957 pgdat->per_cpu_nodestats = &boot_nodestats;
6958
Linus Torvalds1da177e2005-04-16 15:20:36 -07006959 for (j = 0; j < MAX_NR_ZONES; j++) {
6960 struct zone *zone = pgdat->node_zones + j;
Wei Yange6943852018-06-07 17:06:04 -07006961 unsigned long size, freesize, memmap_pages;
Taku Izumid91749c2016-03-15 14:55:18 -07006962 unsigned long zone_start_pfn = zone->zone_start_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006963
Gu Zhengfebd5942015-06-24 16:57:02 -07006964 size = zone->spanned_pages;
Wei Yange6943852018-06-07 17:06:04 -07006965 freesize = zone->present_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006966
Mel Gorman0e0b8642006-09-27 01:49:56 -07006967 /*
Jiang Liu9feedc92012-12-12 13:52:12 -08006968 * Adjust freesize so that it accounts for how much memory
Mel Gorman0e0b8642006-09-27 01:49:56 -07006969 * is used by this zone for memmap. This affects the watermark
6970 * and per-cpu initialisations
6971 */
Wei Yange6943852018-06-07 17:06:04 -07006972 memmap_pages = calc_memmap_size(size, freesize);
Zhong Hongboba914f42014-12-12 16:56:21 -08006973 if (!is_highmem_idx(j)) {
6974 if (freesize >= memmap_pages) {
6975 freesize -= memmap_pages;
6976 if (memmap_pages)
6977 printk(KERN_DEBUG
6978 " %s zone: %lu pages used for memmap\n",
6979 zone_names[j], memmap_pages);
6980 } else
Joe Perches11705322016-03-17 14:19:50 -07006981 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
Zhong Hongboba914f42014-12-12 16:56:21 -08006982 zone_names[j], memmap_pages, freesize);
6983 }
Mel Gorman0e0b8642006-09-27 01:49:56 -07006984
Christoph Lameter62672762007-02-10 01:43:07 -08006985 /* Account for reserved pages */
Jiang Liu9feedc92012-12-12 13:52:12 -08006986 if (j == 0 && freesize > dma_reserve) {
6987 freesize -= dma_reserve;
Yinghai Lud903ef92008-10-18 20:27:06 -07006988 printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
Christoph Lameter62672762007-02-10 01:43:07 -08006989 zone_names[0], dma_reserve);
Mel Gorman0e0b8642006-09-27 01:49:56 -07006990 }
6991
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07006992 if (!is_highmem_idx(j))
Jiang Liu9feedc92012-12-12 13:52:12 -08006993 nr_kernel_pages += freesize;
Jiang Liu01cefae2012-12-12 13:52:19 -08006994 /* Charge for highmem memmap if there are enough kernel pages */
6995 else if (nr_kernel_pages > memmap_pages * 2)
6996 nr_kernel_pages -= memmap_pages;
Jiang Liu9feedc92012-12-12 13:52:12 -08006997 nr_all_pages += freesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006998
Jiang Liu9feedc92012-12-12 13:52:12 -08006999 /*
7000 * Set an approximate value for lowmem here, it will be adjusted
7001 * when the bootmem allocator frees pages into the buddy system.
7002 * And all highmem pages will be managed by the buddy system.
7003 */
Oscar Salvador03e85f92018-08-21 21:53:43 -07007004 zone_init_internals(zone, j, nid, freesize);
Johannes Weiner81c0a2b2013-09-11 14:20:47 -07007005
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09007006 if (!size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007007 continue;
7008
Andrew Morton955c1cd2012-05-29 15:06:31 -07007009 set_pageblock_order();
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09007010 setup_usemap(pgdat, zone, zone_start_pfn, size);
7011 init_currently_empty_zone(zone, zone_start_pfn, size);
Mike Rapoportce6ee462021-06-28 19:33:26 -07007012 arch_memmap_init(size, nid, j, zone_start_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007013 }
7014}
7015
Oscar Salvador0cd842f2017-11-15 17:39:18 -08007016#ifdef CONFIG_FLAT_NODE_MEM_MAP
Fabian Frederickbd721ea2016-08-02 14:03:33 -07007017static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007018{
Tony Luckb0aeba72015-11-10 10:09:47 -08007019 unsigned long __maybe_unused start = 0;
Laura Abbotta1c34a32015-11-05 18:48:46 -08007020 unsigned long __maybe_unused offset = 0;
7021
Linus Torvalds1da177e2005-04-16 15:20:36 -07007022 /* Skip empty nodes */
7023 if (!pgdat->node_spanned_pages)
7024 return;
7025
Tony Luckb0aeba72015-11-10 10:09:47 -08007026 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
7027 offset = pgdat->node_start_pfn - start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007028 /* ia64 gets its own node_mem_map, before this, without bootmem */
7029 if (!pgdat->node_mem_map) {
Tony Luckb0aeba72015-11-10 10:09:47 -08007030 unsigned long size, end;
Andy Whitcroftd41dee32005-06-23 00:07:54 -07007031 struct page *map;
7032
Bob Piccoe984bb42006-05-20 15:00:31 -07007033 /*
7034 * The zone's endpoints aren't required to be MAX_ORDER
7035 * aligned but the node_mem_map endpoints must be in order
7036 * for the buddy allocator to function correctly.
7037 */
Cody P Schafer108bcc92013-02-22 16:35:23 -08007038 end = pgdat_end_pfn(pgdat);
Bob Piccoe984bb42006-05-20 15:00:31 -07007039 end = ALIGN(end, MAX_ORDER_NR_PAGES);
7040 size = (end - start) * sizeof(struct page);
Mike Rapoport26fb3da2019-03-11 23:30:42 -07007041 map = memblock_alloc_node(size, SMP_CACHE_BYTES,
7042 pgdat->node_id);
Mike Rapoport23a70522019-03-05 15:46:43 -08007043 if (!map)
7044 panic("Failed to allocate %ld bytes for node %d memory map\n",
7045 size, pgdat->node_id);
Laura Abbotta1c34a32015-11-05 18:48:46 -08007046 pgdat->node_mem_map = map + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007047 }
Oscar Salvador0cd842f2017-11-15 17:39:18 -08007048 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
7049 __func__, pgdat->node_id, (unsigned long)pgdat,
7050 (unsigned long)pgdat->node_mem_map);
Roman Zippel12d810c2007-05-31 00:40:54 -07007051#ifndef CONFIG_NEED_MULTIPLE_NODES
Linus Torvalds1da177e2005-04-16 15:20:36 -07007052 /*
7053 * With no DISCONTIG, the global mem_map is just set as node 0's
7054 */
Mel Gormanc7132162006-09-27 01:49:43 -07007055 if (pgdat == NODE_DATA(0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007056 mem_map = NODE_DATA(0)->node_mem_map;
Mel Gormanc7132162006-09-27 01:49:43 -07007057 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
Laura Abbotta1c34a32015-11-05 18:48:46 -08007058 mem_map -= offset;
Mel Gormanc7132162006-09-27 01:49:43 -07007059 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007060#endif
7061}
Oscar Salvador0cd842f2017-11-15 17:39:18 -08007062#else
7063static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
7064#endif /* CONFIG_FLAT_NODE_MEM_MAP */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007065
Oscar Salvador0188dc92018-08-21 21:53:39 -07007066#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
7067static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
7068{
Oscar Salvador0188dc92018-08-21 21:53:39 -07007069 pgdat->first_deferred_pfn = ULONG_MAX;
7070}
7071#else
7072static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
7073#endif
7074
Mike Rapoport854e8842020-06-03 15:58:13 -07007075static void __init free_area_init_node(int nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007076{
Johannes Weiner9109fb72008-07-23 21:27:20 -07007077 pg_data_t *pgdat = NODE_DATA(nid);
Zhang Yanfei7960aed2013-07-08 15:59:52 -07007078 unsigned long start_pfn = 0;
7079 unsigned long end_pfn = 0;
Johannes Weiner9109fb72008-07-23 21:27:20 -07007080
Minchan Kim88fdf752012-07-31 16:46:14 -07007081 /* pg_data_t should be reset to zero when it's allocated */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07007082 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
Minchan Kim88fdf752012-07-31 16:46:14 -07007083
Mike Rapoport854e8842020-06-03 15:58:13 -07007084 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07007085
Linus Torvalds1da177e2005-04-16 15:20:36 -07007086 pgdat->node_id = nid;
Mike Rapoport854e8842020-06-03 15:58:13 -07007087 pgdat->node_start_pfn = start_pfn;
Mel Gorman75ef7182016-07-28 15:45:24 -07007088 pgdat->per_cpu_nodestats = NULL;
Mike Rapoport854e8842020-06-03 15:58:13 -07007089
Juergen Gross8d29e182015-02-11 15:26:01 -08007090 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
Zhen Lei4ada0c52015-09-08 15:04:19 -07007091 (u64)start_pfn << PAGE_SHIFT,
7092 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
Mike Rapoport854e8842020-06-03 15:58:13 -07007093 calculate_node_totalpages(pgdat, start_pfn, end_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007094
7095 alloc_node_mem_map(pgdat);
Oscar Salvador0188dc92018-08-21 21:53:39 -07007096 pgdat_set_deferred_range(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007097
Wei Yang7f3eb552015-09-08 14:59:50 -07007098 free_area_init_core(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007099}
7100
Mike Rapoportbc9331a2020-06-03 15:58:09 -07007101void __init free_area_init_memoryless_node(int nid)
Mike Rapoport3f08a302020-06-03 15:57:02 -07007102{
Mike Rapoport854e8842020-06-03 15:58:13 -07007103 free_area_init_node(nid);
Mike Rapoport3f08a302020-06-03 15:57:02 -07007104}
7105
Miklos Szeredi418508c2007-05-23 13:57:55 -07007106#if MAX_NUMNODES > 1
7107/*
7108 * Figure out the number of possible node ids.
7109 */
Cody P Schaferf9872ca2013-04-29 15:08:01 -07007110void __init setup_nr_node_ids(void)
Miklos Szeredi418508c2007-05-23 13:57:55 -07007111{
Wei Yang904a9552015-09-08 14:59:48 -07007112 unsigned int highest;
Miklos Szeredi418508c2007-05-23 13:57:55 -07007113
Wei Yang904a9552015-09-08 14:59:48 -07007114 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
Miklos Szeredi418508c2007-05-23 13:57:55 -07007115 nr_node_ids = highest + 1;
7116}
Miklos Szeredi418508c2007-05-23 13:57:55 -07007117#endif
7118
Mel Gormanc7132162006-09-27 01:49:43 -07007119/**
Tejun Heo1e019792011-07-12 09:45:34 +02007120 * node_map_pfn_alignment - determine the maximum internode alignment
7121 *
7122 * This function should be called after node map is populated and sorted.
7123 * It calculates the maximum power of two alignment which can distinguish
7124 * all the nodes.
7125 *
7126 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
7127 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
7128 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
7129 * shifted, 1GiB is enough and this function will indicate so.
7130 *
7131 * This is used to test whether pfn -> nid mapping of the chosen memory
7132 * model has fine enough granularity to avoid incorrect mapping for the
7133 * populated node map.
7134 *
Mike Rapoporta862f682019-03-05 15:48:42 -08007135 * Return: the determined alignment in pfn's. 0 if there is no alignment
Tejun Heo1e019792011-07-12 09:45:34 +02007136 * requirement (single node).
7137 */
7138unsigned long __init node_map_pfn_alignment(void)
7139{
7140 unsigned long accl_mask = 0, last_end = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02007141 unsigned long start, end, mask;
Anshuman Khandual98fa15f2019-03-05 15:42:58 -08007142 int last_nid = NUMA_NO_NODE;
Tejun Heoc13291a2011-07-12 10:46:30 +02007143 int i, nid;
Tejun Heo1e019792011-07-12 09:45:34 +02007144
Tejun Heoc13291a2011-07-12 10:46:30 +02007145 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
Tejun Heo1e019792011-07-12 09:45:34 +02007146 if (!start || last_nid < 0 || last_nid == nid) {
7147 last_nid = nid;
7148 last_end = end;
7149 continue;
7150 }
7151
7152 /*
7153 * Start with a mask granular enough to pin-point to the
7154 * start pfn and tick off bits one-by-one until it becomes
7155 * too coarse to separate the current node from the last.
7156 */
7157 mask = ~((1 << __ffs(start)) - 1);
7158 while (mask && last_end <= (start & (mask << 1)))
7159 mask <<= 1;
7160
7161 /* accumulate all internode masks */
7162 accl_mask |= mask;
7163 }
7164
7165 /* convert mask to number of pages */
7166 return ~accl_mask + 1;
7167}
7168
Mel Gormanc7132162006-09-27 01:49:43 -07007169/**
7170 * find_min_pfn_with_active_regions - Find the minimum PFN registered
7171 *
Mike Rapoporta862f682019-03-05 15:48:42 -08007172 * Return: the minimum PFN based on information provided via
Zhang Zhen7d018172014-06-04 16:10:53 -07007173 * memblock_set_node().
Mel Gormanc7132162006-09-27 01:49:43 -07007174 */
7175unsigned long __init find_min_pfn_with_active_regions(void)
7176{
Mike Rapoport8a1b25f2020-06-03 15:58:18 -07007177 return PHYS_PFN(memblock_start_of_DRAM());
Mel Gormanc7132162006-09-27 01:49:43 -07007178}
7179
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007180/*
7181 * early_calculate_totalpages()
7182 * Sum pages in active regions for movable zone.
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007183 * Populate N_MEMORY for calculating usable_nodes.
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007184 */
Adrian Bunk484f51f2007-10-16 01:26:03 -07007185static unsigned long __init early_calculate_totalpages(void)
Mel Gorman7e63efef2007-07-17 04:03:15 -07007186{
Mel Gorman7e63efef2007-07-17 04:03:15 -07007187 unsigned long totalpages = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02007188 unsigned long start_pfn, end_pfn;
7189 int i, nid;
Mel Gorman7e63efef2007-07-17 04:03:15 -07007190
Tejun Heoc13291a2011-07-12 10:46:30 +02007191 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
7192 unsigned long pages = end_pfn - start_pfn;
7193
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007194 totalpages += pages;
7195 if (pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007196 node_set_state(nid, N_MEMORY);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007197 }
Pintu Kumarb8af2942013-09-11 14:20:34 -07007198 return totalpages;
Mel Gorman7e63efef2007-07-17 04:03:15 -07007199}
7200
Mel Gorman2a1e2742007-07-17 04:03:12 -07007201/*
7202 * Find the PFN the Movable zone begins in each node. Kernel memory
7203 * is spread evenly between nodes as long as the nodes have enough
7204 * memory. When they don't, some nodes will have more kernelcore than
7205 * others
7206 */
Kautuk Consulb224ef82012-03-21 16:34:15 -07007207static void __init find_zone_movable_pfns_for_nodes(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007208{
7209 int i, nid;
7210 unsigned long usable_startpfn;
7211 unsigned long kernelcore_node, kernelcore_remaining;
Yinghai Lu66918dc2009-06-30 11:41:37 -07007212 /* save the state before borrow the nodemask */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007213 nodemask_t saved_node_state = node_states[N_MEMORY];
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007214 unsigned long totalpages = early_calculate_totalpages();
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007215 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
Emil Medve136199f2014-04-07 15:37:52 -07007216 struct memblock_region *r;
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007217
7218 /* Need to find movable_zone earlier when movable_node is specified. */
7219 find_usable_zone_for_movable();
Mel Gorman2a1e2742007-07-17 04:03:12 -07007220
Mel Gorman7e63efef2007-07-17 04:03:15 -07007221 /*
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007222 * If movable_node is specified, ignore kernelcore and movablecore
7223 * options.
7224 */
7225 if (movable_node_is_enabled()) {
Mike Rapoportcc6de162020-10-13 16:58:30 -07007226 for_each_mem_region(r) {
Emil Medve136199f2014-04-07 15:37:52 -07007227 if (!memblock_is_hotpluggable(r))
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007228 continue;
7229
Mike Rapoportd622abf2020-06-03 15:56:53 -07007230 nid = memblock_get_region_node(r);
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007231
Emil Medve136199f2014-04-07 15:37:52 -07007232 usable_startpfn = PFN_DOWN(r->base);
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007233 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7234 min(usable_startpfn, zone_movable_pfn[nid]) :
7235 usable_startpfn;
7236 }
7237
7238 goto out2;
7239 }
7240
7241 /*
Taku Izumi342332e2016-03-15 14:55:22 -07007242 * If kernelcore=mirror is specified, ignore movablecore option
7243 */
7244 if (mirrored_kernelcore) {
7245 bool mem_below_4gb_not_mirrored = false;
7246
Mike Rapoportcc6de162020-10-13 16:58:30 -07007247 for_each_mem_region(r) {
Taku Izumi342332e2016-03-15 14:55:22 -07007248 if (memblock_is_mirror(r))
7249 continue;
7250
Mike Rapoportd622abf2020-06-03 15:56:53 -07007251 nid = memblock_get_region_node(r);
Taku Izumi342332e2016-03-15 14:55:22 -07007252
7253 usable_startpfn = memblock_region_memory_base_pfn(r);
7254
7255 if (usable_startpfn < 0x100000) {
7256 mem_below_4gb_not_mirrored = true;
7257 continue;
7258 }
7259
7260 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7261 min(usable_startpfn, zone_movable_pfn[nid]) :
7262 usable_startpfn;
7263 }
7264
7265 if (mem_below_4gb_not_mirrored)
Chen Tao633bf2f2020-06-03 16:00:02 -07007266 pr_warn("This configuration results in unmirrored kernel memory.\n");
Taku Izumi342332e2016-03-15 14:55:22 -07007267
7268 goto out2;
7269 }
7270
7271 /*
David Rientjesa5c6d652018-04-05 16:23:09 -07007272 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
7273 * amount of necessary memory.
7274 */
7275 if (required_kernelcore_percent)
7276 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
7277 10000UL;
7278 if (required_movablecore_percent)
7279 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
7280 10000UL;
7281
7282 /*
7283 * If movablecore= was specified, calculate what size of
Mel Gorman7e63efef2007-07-17 04:03:15 -07007284 * kernelcore that corresponds so that memory usable for
7285 * any allocation type is evenly spread. If both kernelcore
7286 * and movablecore are specified, then the value of kernelcore
7287 * will be used for required_kernelcore if it's greater than
7288 * what movablecore would have allowed.
7289 */
7290 if (required_movablecore) {
Mel Gorman7e63efef2007-07-17 04:03:15 -07007291 unsigned long corepages;
7292
7293 /*
7294 * Round-up so that ZONE_MOVABLE is at least as large as what
7295 * was requested by the user
7296 */
7297 required_movablecore =
7298 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
Xishi Qiu9fd745d2015-11-05 18:48:11 -08007299 required_movablecore = min(totalpages, required_movablecore);
Mel Gorman7e63efef2007-07-17 04:03:15 -07007300 corepages = totalpages - required_movablecore;
7301
7302 required_kernelcore = max(required_kernelcore, corepages);
7303 }
7304
Xishi Qiubde304b2015-11-05 18:48:56 -08007305 /*
7306 * If kernelcore was not specified or kernelcore size is larger
7307 * than totalpages, there is no ZONE_MOVABLE.
7308 */
7309 if (!required_kernelcore || required_kernelcore >= totalpages)
Yinghai Lu66918dc2009-06-30 11:41:37 -07007310 goto out;
Mel Gorman2a1e2742007-07-17 04:03:12 -07007311
7312 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
Mel Gorman2a1e2742007-07-17 04:03:12 -07007313 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
7314
7315restart:
7316 /* Spread kernelcore memory as evenly as possible throughout nodes */
7317 kernelcore_node = required_kernelcore / usable_nodes;
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007318 for_each_node_state(nid, N_MEMORY) {
Tejun Heoc13291a2011-07-12 10:46:30 +02007319 unsigned long start_pfn, end_pfn;
7320
Mel Gorman2a1e2742007-07-17 04:03:12 -07007321 /*
7322 * Recalculate kernelcore_node if the division per node
7323 * now exceeds what is necessary to satisfy the requested
7324 * amount of memory for the kernel
7325 */
7326 if (required_kernelcore < kernelcore_node)
7327 kernelcore_node = required_kernelcore / usable_nodes;
7328
7329 /*
7330 * As the map is walked, we track how much memory is usable
7331 * by the kernel using kernelcore_remaining. When it is
7332 * 0, the rest of the node is usable by ZONE_MOVABLE
7333 */
7334 kernelcore_remaining = kernelcore_node;
7335
7336 /* Go through each range of PFNs within this node */
Tejun Heoc13291a2011-07-12 10:46:30 +02007337 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
Mel Gorman2a1e2742007-07-17 04:03:12 -07007338 unsigned long size_pages;
7339
Tejun Heoc13291a2011-07-12 10:46:30 +02007340 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007341 if (start_pfn >= end_pfn)
7342 continue;
7343
7344 /* Account for what is only usable for kernelcore */
7345 if (start_pfn < usable_startpfn) {
7346 unsigned long kernel_pages;
7347 kernel_pages = min(end_pfn, usable_startpfn)
7348 - start_pfn;
7349
7350 kernelcore_remaining -= min(kernel_pages,
7351 kernelcore_remaining);
7352 required_kernelcore -= min(kernel_pages,
7353 required_kernelcore);
7354
7355 /* Continue if range is now fully accounted */
7356 if (end_pfn <= usable_startpfn) {
7357
7358 /*
7359 * Push zone_movable_pfn to the end so
7360 * that if we have to rebalance
7361 * kernelcore across nodes, we will
7362 * not double account here
7363 */
7364 zone_movable_pfn[nid] = end_pfn;
7365 continue;
7366 }
7367 start_pfn = usable_startpfn;
7368 }
7369
7370 /*
7371 * The usable PFN range for ZONE_MOVABLE is from
7372 * start_pfn->end_pfn. Calculate size_pages as the
7373 * number of pages used as kernelcore
7374 */
7375 size_pages = end_pfn - start_pfn;
7376 if (size_pages > kernelcore_remaining)
7377 size_pages = kernelcore_remaining;
7378 zone_movable_pfn[nid] = start_pfn + size_pages;
7379
7380 /*
7381 * Some kernelcore has been met, update counts and
7382 * break if the kernelcore for this node has been
Pintu Kumarb8af2942013-09-11 14:20:34 -07007383 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07007384 */
7385 required_kernelcore -= min(required_kernelcore,
7386 size_pages);
7387 kernelcore_remaining -= size_pages;
7388 if (!kernelcore_remaining)
7389 break;
7390 }
7391 }
7392
7393 /*
7394 * If there is still required_kernelcore, we do another pass with one
7395 * less node in the count. This will push zone_movable_pfn[nid] further
7396 * along on the nodes that still have memory until kernelcore is
Pintu Kumarb8af2942013-09-11 14:20:34 -07007397 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07007398 */
7399 usable_nodes--;
7400 if (usable_nodes && required_kernelcore > usable_nodes)
7401 goto restart;
7402
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007403out2:
Mel Gorman2a1e2742007-07-17 04:03:12 -07007404 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
Alistair Popple7188e7c2022-03-22 14:43:26 -07007405 for (nid = 0; nid < MAX_NUMNODES; nid++) {
7406 unsigned long start_pfn, end_pfn;
7407
Mel Gorman2a1e2742007-07-17 04:03:12 -07007408 zone_movable_pfn[nid] =
7409 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
Yinghai Lu66918dc2009-06-30 11:41:37 -07007410
Alistair Popple7188e7c2022-03-22 14:43:26 -07007411 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
7412 if (zone_movable_pfn[nid] >= end_pfn)
7413 zone_movable_pfn[nid] = 0;
7414 }
7415
Yinghai Lu20e69262013-03-01 14:51:27 -08007416out:
Yinghai Lu66918dc2009-06-30 11:41:37 -07007417 /* restore the node_state */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007418 node_states[N_MEMORY] = saved_node_state;
Mel Gorman2a1e2742007-07-17 04:03:12 -07007419}
7420
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007421/* Any regular or high memory on that node ? */
7422static void check_for_memory(pg_data_t *pgdat, int nid)
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007423{
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007424 enum zone_type zone_type;
7425
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007426 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007427 struct zone *zone = &pgdat->node_zones[zone_type];
Xishi Qiub38a8722013-11-12 15:07:20 -08007428 if (populated_zone(zone)) {
Oscar Salvador7b0e0c02018-10-26 15:03:58 -07007429 if (IS_ENABLED(CONFIG_HIGHMEM))
7430 node_set_state(nid, N_HIGH_MEMORY);
7431 if (zone_type <= ZONE_NORMAL)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007432 node_set_state(nid, N_NORMAL_MEMORY);
Bob Liud0048b02012-01-12 17:19:07 -08007433 break;
7434 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007435 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007436}
7437
Mike Rapoport51930df2020-06-03 15:58:03 -07007438/*
7439 * Some architecturs, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
7440 * such cases we allow max_zone_pfn sorted in the descending order
7441 */
7442bool __weak arch_has_descending_max_zone_pfns(void)
7443{
7444 return false;
7445}
7446
Mel Gormanc7132162006-09-27 01:49:43 -07007447/**
Mike Rapoport9691a072020-06-03 15:57:10 -07007448 * free_area_init - Initialise all pg_data_t and zone data
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007449 * @max_zone_pfn: an array of max PFNs for each zone
Mel Gormanc7132162006-09-27 01:49:43 -07007450 *
7451 * This will call free_area_init_node() for each active node in the system.
Zhang Zhen7d018172014-06-04 16:10:53 -07007452 * Using the page ranges provided by memblock_set_node(), the size of each
Mel Gormanc7132162006-09-27 01:49:43 -07007453 * zone in each node and their holes is calculated. If the maximum PFN
7454 * between two adjacent zones match, it is assumed that the zone is empty.
7455 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
7456 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
7457 * starts where the previous one ended. For example, ZONE_DMA32 starts
7458 * at arch_max_dma_pfn.
7459 */
Mike Rapoport9691a072020-06-03 15:57:10 -07007460void __init free_area_init(unsigned long *max_zone_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07007461{
Tejun Heoc13291a2011-07-12 10:46:30 +02007462 unsigned long start_pfn, end_pfn;
Mike Rapoport51930df2020-06-03 15:58:03 -07007463 int i, nid, zone;
7464 bool descending;
Mel Gormana6af2bc2007-02-10 01:42:57 -08007465
Mel Gormanc7132162006-09-27 01:49:43 -07007466 /* Record where the zone boundaries are */
7467 memset(arch_zone_lowest_possible_pfn, 0,
7468 sizeof(arch_zone_lowest_possible_pfn));
7469 memset(arch_zone_highest_possible_pfn, 0,
7470 sizeof(arch_zone_highest_possible_pfn));
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007471
7472 start_pfn = find_min_pfn_with_active_regions();
Mike Rapoport51930df2020-06-03 15:58:03 -07007473 descending = arch_has_descending_max_zone_pfns();
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007474
7475 for (i = 0; i < MAX_NR_ZONES; i++) {
Mike Rapoport51930df2020-06-03 15:58:03 -07007476 if (descending)
7477 zone = MAX_NR_ZONES - i - 1;
7478 else
7479 zone = i;
7480
7481 if (zone == ZONE_MOVABLE)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007482 continue;
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007483
Mike Rapoport51930df2020-06-03 15:58:03 -07007484 end_pfn = max(max_zone_pfn[zone], start_pfn);
7485 arch_zone_lowest_possible_pfn[zone] = start_pfn;
7486 arch_zone_highest_possible_pfn[zone] = end_pfn;
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007487
7488 start_pfn = end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07007489 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07007490
7491 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
7492 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
Kautuk Consulb224ef82012-03-21 16:34:15 -07007493 find_zone_movable_pfns_for_nodes();
Mel Gormanc7132162006-09-27 01:49:43 -07007494
Mel Gormanc7132162006-09-27 01:49:43 -07007495 /* Print out the zone ranges */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007496 pr_info("Zone ranges:\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07007497 for (i = 0; i < MAX_NR_ZONES; i++) {
7498 if (i == ZONE_MOVABLE)
7499 continue;
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007500 pr_info(" %-8s ", zone_names[i]);
David Rientjes72f0ba02010-03-05 13:42:14 -08007501 if (arch_zone_lowest_possible_pfn[i] ==
7502 arch_zone_highest_possible_pfn[i])
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007503 pr_cont("empty\n");
David Rientjes72f0ba02010-03-05 13:42:14 -08007504 else
Juergen Gross8d29e182015-02-11 15:26:01 -08007505 pr_cont("[mem %#018Lx-%#018Lx]\n",
7506 (u64)arch_zone_lowest_possible_pfn[i]
7507 << PAGE_SHIFT,
7508 ((u64)arch_zone_highest_possible_pfn[i]
Bjorn Helgaasa62e2f42012-05-29 15:06:30 -07007509 << PAGE_SHIFT) - 1);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007510 }
7511
7512 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007513 pr_info("Movable zone start for each node\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07007514 for (i = 0; i < MAX_NUMNODES; i++) {
7515 if (zone_movable_pfn[i])
Juergen Gross8d29e182015-02-11 15:26:01 -08007516 pr_info(" Node %d: %#018Lx\n", i,
7517 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007518 }
Mel Gormanc7132162006-09-27 01:49:43 -07007519
Dan Williamsf46edbd2019-07-18 15:58:04 -07007520 /*
7521 * Print out the early node map, and initialize the
7522 * subsection-map relative to active online memory ranges to
7523 * enable future "sub-section" extensions of the memory map.
7524 */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007525 pr_info("Early memory node ranges\n");
Dan Williamsf46edbd2019-07-18 15:58:04 -07007526 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
Juergen Gross8d29e182015-02-11 15:26:01 -08007527 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
7528 (u64)start_pfn << PAGE_SHIFT,
7529 ((u64)end_pfn << PAGE_SHIFT) - 1);
Dan Williamsf46edbd2019-07-18 15:58:04 -07007530 subsection_map_init(start_pfn, end_pfn - start_pfn);
7531 }
Mel Gormanc7132162006-09-27 01:49:43 -07007532
7533 /* Initialise every node */
Mel Gorman708614e2008-07-23 21:26:51 -07007534 mminit_verify_pageflags_layout();
Christoph Lameter8ef82862007-02-20 13:57:52 -08007535 setup_nr_node_ids();
Mel Gormanc7132162006-09-27 01:49:43 -07007536 for_each_online_node(nid) {
7537 pg_data_t *pgdat = NODE_DATA(nid);
Mike Rapoport854e8842020-06-03 15:58:13 -07007538 free_area_init_node(nid);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007539
7540 /* Any memory on that node */
7541 if (pgdat->node_present_pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007542 node_set_state(nid, N_MEMORY);
7543 check_for_memory(pgdat, nid);
Mel Gormanc7132162006-09-27 01:49:43 -07007544 }
Mike Rapoportce6ee462021-06-28 19:33:26 -07007545
7546 memmap_init();
Mel Gormanc7132162006-09-27 01:49:43 -07007547}
Mel Gorman2a1e2742007-07-17 04:03:12 -07007548
David Rientjesa5c6d652018-04-05 16:23:09 -07007549static int __init cmdline_parse_core(char *p, unsigned long *core,
7550 unsigned long *percent)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007551{
7552 unsigned long long coremem;
David Rientjesa5c6d652018-04-05 16:23:09 -07007553 char *endptr;
7554
Mel Gorman2a1e2742007-07-17 04:03:12 -07007555 if (!p)
7556 return -EINVAL;
7557
David Rientjesa5c6d652018-04-05 16:23:09 -07007558 /* Value may be a percentage of total memory, otherwise bytes */
7559 coremem = simple_strtoull(p, &endptr, 0);
7560 if (*endptr == '%') {
7561 /* Paranoid check for percent values greater than 100 */
7562 WARN_ON(coremem > 100);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007563
David Rientjesa5c6d652018-04-05 16:23:09 -07007564 *percent = coremem;
7565 } else {
7566 coremem = memparse(p, &p);
7567 /* Paranoid check that UL is enough for the coremem value */
7568 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007569
David Rientjesa5c6d652018-04-05 16:23:09 -07007570 *core = coremem >> PAGE_SHIFT;
7571 *percent = 0UL;
7572 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07007573 return 0;
7574}
Mel Gormaned7ed362007-07-17 04:03:14 -07007575
Mel Gorman7e63efef2007-07-17 04:03:15 -07007576/*
7577 * kernelcore=size sets the amount of memory for use for allocations that
7578 * cannot be reclaimed or migrated.
7579 */
7580static int __init cmdline_parse_kernelcore(char *p)
7581{
Taku Izumi342332e2016-03-15 14:55:22 -07007582 /* parse kernelcore=mirror */
7583 if (parse_option_str(p, "mirror")) {
7584 mirrored_kernelcore = true;
7585 return 0;
7586 }
7587
David Rientjesa5c6d652018-04-05 16:23:09 -07007588 return cmdline_parse_core(p, &required_kernelcore,
7589 &required_kernelcore_percent);
Mel Gorman7e63efef2007-07-17 04:03:15 -07007590}
7591
7592/*
7593 * movablecore=size sets the amount of memory for use for allocations that
7594 * can be reclaimed or migrated.
7595 */
7596static int __init cmdline_parse_movablecore(char *p)
7597{
David Rientjesa5c6d652018-04-05 16:23:09 -07007598 return cmdline_parse_core(p, &required_movablecore,
7599 &required_movablecore_percent);
Mel Gorman7e63efef2007-07-17 04:03:15 -07007600}
7601
Mel Gormaned7ed362007-07-17 04:03:14 -07007602early_param("kernelcore", cmdline_parse_kernelcore);
Mel Gorman7e63efef2007-07-17 04:03:15 -07007603early_param("movablecore", cmdline_parse_movablecore);
Mel Gormaned7ed362007-07-17 04:03:14 -07007604
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07007605void adjust_managed_page_count(struct page *page, long count)
7606{
Arun KS9705bea2018-12-28 00:34:24 -08007607 atomic_long_add(count, &page_zone(page)->managed_pages);
Arun KSca79b0c2018-12-28 00:34:29 -08007608 totalram_pages_add(count);
Jiang Liu3dcc0572013-07-03 15:03:21 -07007609#ifdef CONFIG_HIGHMEM
7610 if (PageHighMem(page))
Arun KSca79b0c2018-12-28 00:34:29 -08007611 totalhigh_pages_add(count);
Jiang Liu3dcc0572013-07-03 15:03:21 -07007612#endif
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07007613}
Jiang Liu3dcc0572013-07-03 15:03:21 -07007614EXPORT_SYMBOL(adjust_managed_page_count);
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07007615
Alexey Dobriyane5cb1132018-12-28 00:36:03 -08007616unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
Jiang Liu69afade2013-04-29 15:06:21 -07007617{
Jiang Liu11199692013-07-03 15:02:48 -07007618 void *pos;
7619 unsigned long pages = 0;
Jiang Liu69afade2013-04-29 15:06:21 -07007620
Jiang Liu11199692013-07-03 15:02:48 -07007621 start = (void *)PAGE_ALIGN((unsigned long)start);
7622 end = (void *)((unsigned long)end & PAGE_MASK);
7623 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
Dave Hansen0d834322018-08-02 15:58:26 -07007624 struct page *page = virt_to_page(pos);
7625 void *direct_map_addr;
7626
7627 /*
7628 * 'direct_map_addr' might be different from 'pos'
7629 * because some architectures' virt_to_page()
7630 * work with aliases. Getting the direct map
7631 * address ensures that we get a _writeable_
7632 * alias for the memset().
7633 */
7634 direct_map_addr = page_address(page);
Jiang Liudbe67df2013-07-03 15:02:51 -07007635 if ((unsigned int)poison <= 0xFF)
Dave Hansen0d834322018-08-02 15:58:26 -07007636 memset(direct_map_addr, poison, PAGE_SIZE);
7637
7638 free_reserved_page(page);
Jiang Liu69afade2013-04-29 15:06:21 -07007639 }
7640
7641 if (pages && s)
Josh Poimboeufadb1fe92016-10-25 09:51:14 -05007642 pr_info("Freeing %s memory: %ldK\n",
7643 s, pages << (PAGE_SHIFT - 10));
Jiang Liu69afade2013-04-29 15:06:21 -07007644
7645 return pages;
7646}
7647
Jiang Liucfa11e02013-04-29 15:07:00 -07007648#ifdef CONFIG_HIGHMEM
7649void free_highmem_page(struct page *page)
7650{
7651 __free_reserved_page(page);
Arun KSca79b0c2018-12-28 00:34:29 -08007652 totalram_pages_inc();
Arun KS9705bea2018-12-28 00:34:24 -08007653 atomic_long_inc(&page_zone(page)->managed_pages);
Arun KSca79b0c2018-12-28 00:34:29 -08007654 totalhigh_pages_inc();
Jiang Liucfa11e02013-04-29 15:07:00 -07007655}
7656#endif
7657
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007658
7659void __init mem_init_print_info(const char *str)
7660{
7661 unsigned long physpages, codesize, datasize, rosize, bss_size;
7662 unsigned long init_code_size, init_data_size;
7663
7664 physpages = get_num_physpages();
7665 codesize = _etext - _stext;
7666 datasize = _edata - _sdata;
7667 rosize = __end_rodata - __start_rodata;
7668 bss_size = __bss_stop - __bss_start;
7669 init_data_size = __init_end - __init_begin;
7670 init_code_size = _einittext - _sinittext;
7671
7672 /*
7673 * Detect special cases and adjust section sizes accordingly:
7674 * 1) .init.* may be embedded into .data sections
7675 * 2) .init.text.* may be out of [__init_begin, __init_end],
7676 * please refer to arch/tile/kernel/vmlinux.lds.S.
7677 * 3) .rodata.* may be embedded into .text or .data sections.
7678 */
7679#define adj_init_size(start, end, size, pos, adj) \
Pintu Kumarb8af2942013-09-11 14:20:34 -07007680 do { \
Xiongwei Song69848f92022-01-14 14:07:24 -08007681 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
Pintu Kumarb8af2942013-09-11 14:20:34 -07007682 size -= adj; \
7683 } while (0)
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007684
7685 adj_init_size(__init_begin, __init_end, init_data_size,
7686 _sinittext, init_code_size);
7687 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
7688 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
7689 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
7690 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
7691
7692#undef adj_init_size
7693
Joe Perches756a0252016-03-17 14:19:47 -07007694 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007695#ifdef CONFIG_HIGHMEM
Joe Perches756a0252016-03-17 14:19:47 -07007696 ", %luK highmem"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007697#endif
Joe Perches756a0252016-03-17 14:19:47 -07007698 "%s%s)\n",
7699 nr_free_pages() << (PAGE_SHIFT - 10),
7700 physpages << (PAGE_SHIFT - 10),
7701 codesize >> 10, datasize >> 10, rosize >> 10,
7702 (init_data_size + init_code_size) >> 10, bss_size >> 10,
Arun KSca79b0c2018-12-28 00:34:29 -08007703 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),
Joe Perches756a0252016-03-17 14:19:47 -07007704 totalcma_pages << (PAGE_SHIFT - 10),
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007705#ifdef CONFIG_HIGHMEM
Arun KSca79b0c2018-12-28 00:34:29 -08007706 totalhigh_pages() << (PAGE_SHIFT - 10),
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007707#endif
Joe Perches756a0252016-03-17 14:19:47 -07007708 str ? ", " : "", str ? str : "");
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007709}
7710
Mel Gorman0e0b8642006-09-27 01:49:56 -07007711/**
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007712 * set_dma_reserve - set the specified number of pages reserved in the first zone
7713 * @new_dma_reserve: The number of pages to mark reserved
Mel Gorman0e0b8642006-09-27 01:49:56 -07007714 *
Yaowei Bai013110a2015-09-08 15:04:10 -07007715 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
Mel Gorman0e0b8642006-09-27 01:49:56 -07007716 * In the DMA zone, a significant percentage may be consumed by kernel image
7717 * and other unfreeable allocations which can skew the watermarks badly. This
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007718 * function may optionally be used to account for unfreeable pages in the
7719 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
7720 * smaller per-cpu batchsize.
Mel Gorman0e0b8642006-09-27 01:49:56 -07007721 */
7722void __init set_dma_reserve(unsigned long new_dma_reserve)
7723{
7724 dma_reserve = new_dma_reserve;
7725}
7726
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007727static int page_alloc_cpu_dead(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007728{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007729
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007730 lru_add_drain_cpu(cpu);
7731 drain_pages(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08007732
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007733 /*
7734 * Spill the event counters of the dead processor
7735 * into the current processors event counters.
7736 * This artificially elevates the count of the current
7737 * processor.
7738 */
7739 vm_events_fold_cpu(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08007740
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007741 /*
7742 * Zero the differential counters of the dead processor
7743 * so that the vm statistics are consistent.
7744 *
7745 * This is only okay since the processor is dead and cannot
7746 * race with what we are doing.
7747 */
7748 cpu_vm_stats_fold(cpu);
7749 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007750}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007751
Nicholas Piggine03a5122019-07-11 20:59:12 -07007752#ifdef CONFIG_NUMA
7753int hashdist = HASHDIST_DEFAULT;
7754
7755static int __init set_hashdist(char *str)
7756{
7757 if (!str)
7758 return 0;
7759 hashdist = simple_strtoul(str, &str, 0);
7760 return 1;
7761}
7762__setup("hashdist=", set_hashdist);
7763#endif
7764
Linus Torvalds1da177e2005-04-16 15:20:36 -07007765void __init page_alloc_init(void)
7766{
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007767 int ret;
7768
Nicholas Piggine03a5122019-07-11 20:59:12 -07007769#ifdef CONFIG_NUMA
7770 if (num_node_state(N_MEMORY) == 1)
7771 hashdist = 0;
7772#endif
7773
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007774 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
7775 "mm/page_alloc:dead", NULL,
7776 page_alloc_cpu_dead);
7777 WARN_ON(ret < 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007778}
7779
7780/*
Yaowei Bai34b10062015-09-08 15:04:13 -07007781 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007782 * or min_free_kbytes changes.
7783 */
7784static void calculate_totalreserve_pages(void)
7785{
7786 struct pglist_data *pgdat;
7787 unsigned long reserve_pages = 0;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07007788 enum zone_type i, j;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007789
7790 for_each_online_pgdat(pgdat) {
Mel Gorman281e3722016-07-28 15:46:11 -07007791
7792 pgdat->totalreserve_pages = 0;
7793
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007794 for (i = 0; i < MAX_NR_ZONES; i++) {
7795 struct zone *zone = pgdat->node_zones + i;
Mel Gorman3484b2d2014-08-06 16:07:14 -07007796 long max = 0;
Arun KS9705bea2018-12-28 00:34:24 -08007797 unsigned long managed_pages = zone_managed_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007798
7799 /* Find valid and maximum lowmem_reserve in the zone */
7800 for (j = i; j < MAX_NR_ZONES; j++) {
7801 if (zone->lowmem_reserve[j] > max)
7802 max = zone->lowmem_reserve[j];
7803 }
7804
Mel Gorman41858962009-06-16 15:32:12 -07007805 /* we treat the high watermark as reserved pages. */
7806 max += high_wmark_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007807
Arun KS3d6357d2018-12-28 00:34:20 -08007808 if (max > managed_pages)
7809 max = managed_pages;
Johannes Weinera8d01432016-01-14 15:20:15 -08007810
Mel Gorman281e3722016-07-28 15:46:11 -07007811 pgdat->totalreserve_pages += max;
Johannes Weinera8d01432016-01-14 15:20:15 -08007812
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007813 reserve_pages += max;
7814 }
7815 }
7816 totalreserve_pages = reserve_pages;
7817}
7818
7819/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07007820 * setup_per_zone_lowmem_reserve - called whenever
Yaowei Bai34b10062015-09-08 15:04:13 -07007821 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
Linus Torvalds1da177e2005-04-16 15:20:36 -07007822 * has a correct pages reserved value, so an adequate number of
7823 * pages are left in the zone after a successful __alloc_pages().
7824 */
7825static void setup_per_zone_lowmem_reserve(void)
7826{
7827 struct pglist_data *pgdat;
Lorenzo Stoakesd7deea32020-12-14 19:11:22 -08007828 enum zone_type i, j;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007829
KAMEZAWA Hiroyukiec936fc2006-03-27 01:15:59 -08007830 for_each_online_pgdat(pgdat) {
Lorenzo Stoakesd7deea32020-12-14 19:11:22 -08007831 for (i = 0; i < MAX_NR_ZONES - 1; i++) {
7832 struct zone *zone = &pgdat->node_zones[i];
7833 int ratio = sysctl_lowmem_reserve_ratio[i];
7834 bool clear = !ratio || !zone_managed_pages(zone);
7835 unsigned long managed_pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007836
Lorenzo Stoakesd7deea32020-12-14 19:11:22 -08007837 for (j = i + 1; j < MAX_NR_ZONES; j++) {
Liu Shixin10f32b82021-06-28 19:42:33 -07007838 struct zone *upper_zone = &pgdat->node_zones[j];
Lorenzo Stoakesd7deea32020-12-14 19:11:22 -08007839
Liu Shixin10f32b82021-06-28 19:42:33 -07007840 managed_pages += zone_managed_pages(upper_zone);
7841
7842 if (clear)
7843 zone->lowmem_reserve[j] = 0;
7844 else
Lorenzo Stoakesd7deea32020-12-14 19:11:22 -08007845 zone->lowmem_reserve[j] = managed_pages / ratio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007846 }
7847 }
7848 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007849
7850 /* update totalreserve_pages */
7851 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007852}
7853
Mel Gormancfd3da12011-04-25 21:36:42 +00007854static void __setup_per_zone_wmarks(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007855{
7856 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
7857 unsigned long lowmem_pages = 0;
7858 struct zone *zone;
7859 unsigned long flags;
7860
7861 /* Calculate total number of !ZONE_HIGHMEM pages */
7862 for_each_zone(zone) {
7863 if (!is_highmem(zone))
Arun KS9705bea2018-12-28 00:34:24 -08007864 lowmem_pages += zone_managed_pages(zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007865 }
7866
7867 for_each_zone(zone) {
Andrew Mortonac924c62006-05-15 09:43:59 -07007868 u64 tmp;
7869
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07007870 spin_lock_irqsave(&zone->lock, flags);
Arun KS9705bea2018-12-28 00:34:24 -08007871 tmp = (u64)pages_min * zone_managed_pages(zone);
Andrew Mortonac924c62006-05-15 09:43:59 -07007872 do_div(tmp, lowmem_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007873 if (is_highmem(zone)) {
7874 /*
Nick Piggin669ed172005-11-13 16:06:45 -08007875 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
7876 * need highmem pages, so cap pages_min to a small
7877 * value here.
7878 *
Mel Gorman41858962009-06-16 15:32:12 -07007879 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
Wei Yang8bb4e7a2019-03-05 15:46:22 -08007880 * deltas control async page reclaim, and so should
Nick Piggin669ed172005-11-13 16:06:45 -08007881 * not be capped for highmem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007882 */
Andrew Morton90ae8d62013-02-22 16:32:22 -08007883 unsigned long min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007884
Arun KS9705bea2018-12-28 00:34:24 -08007885 min_pages = zone_managed_pages(zone) / 1024;
Andrew Morton90ae8d62013-02-22 16:32:22 -08007886 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
Mel Gormana9214442018-12-28 00:35:44 -08007887 zone->_watermark[WMARK_MIN] = min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007888 } else {
Nick Piggin669ed172005-11-13 16:06:45 -08007889 /*
7890 * If it's a lowmem zone, reserve a number of pages
Linus Torvalds1da177e2005-04-16 15:20:36 -07007891 * proportionate to the zone's size.
7892 */
Mel Gormana9214442018-12-28 00:35:44 -08007893 zone->_watermark[WMARK_MIN] = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007894 }
7895
Johannes Weiner795ae7a2016-03-17 14:19:14 -07007896 /*
7897 * Set the kswapd watermarks distance according to the
7898 * scale factor in proportion to available memory, but
7899 * ensure a minimum size on small systems.
7900 */
7901 tmp = max_t(u64, tmp >> 2,
Arun KS9705bea2018-12-28 00:34:24 -08007902 mult_frac(zone_managed_pages(zone),
Johannes Weiner795ae7a2016-03-17 14:19:14 -07007903 watermark_scale_factor, 10000));
7904
Charan Teja Reddyaa092592020-06-03 15:59:14 -07007905 zone->watermark_boost = 0;
Mel Gormana9214442018-12-28 00:35:44 -08007906 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
7907 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
Marek Szyprowski49f223a2012-01-25 12:49:24 +01007908
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07007909 spin_unlock_irqrestore(&zone->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007910 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007911
7912 /* update totalreserve_pages */
7913 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007914}
7915
Mel Gormancfd3da12011-04-25 21:36:42 +00007916/**
7917 * setup_per_zone_wmarks - called when min_free_kbytes changes
7918 * or when memory is hot-{added|removed}
7919 *
7920 * Ensures that the watermark[min,low,high] values for each zone are set
7921 * correctly with respect to min_free_kbytes.
7922 */
7923void setup_per_zone_wmarks(void)
7924{
Michal Hockob93e0f32017-09-06 16:20:37 -07007925 static DEFINE_SPINLOCK(lock);
7926
7927 spin_lock(&lock);
Mel Gormancfd3da12011-04-25 21:36:42 +00007928 __setup_per_zone_wmarks();
Michal Hockob93e0f32017-09-06 16:20:37 -07007929 spin_unlock(&lock);
Mel Gormancfd3da12011-04-25 21:36:42 +00007930}
7931
Randy Dunlap55a44622009-09-21 17:01:20 -07007932/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07007933 * Initialise min_free_kbytes.
7934 *
7935 * For small machines we want it small (128k min). For large machines
Joel Savitz8beeae82020-07-03 15:15:30 -07007936 * we want it large (256MB max). But it is not linear, because network
Linus Torvalds1da177e2005-04-16 15:20:36 -07007937 * bandwidth does not increase linearly with machine size. We use
7938 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07007939 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
Linus Torvalds1da177e2005-04-16 15:20:36 -07007940 * min_free_kbytes = sqrt(lowmem_kbytes * 16)
7941 *
7942 * which yields
7943 *
7944 * 16MB: 512k
7945 * 32MB: 724k
7946 * 64MB: 1024k
7947 * 128MB: 1448k
7948 * 256MB: 2048k
7949 * 512MB: 2896k
7950 * 1024MB: 4096k
7951 * 2048MB: 5792k
7952 * 4096MB: 8192k
7953 * 8192MB: 11584k
7954 * 16384MB: 16384k
7955 */
KOSAKI Motohiro1b79acc2011-05-24 17:11:32 -07007956int __meminit init_per_zone_wmark_min(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007957{
7958 unsigned long lowmem_kbytes;
Michal Hocko5f127332013-07-08 16:00:40 -07007959 int new_min_free_kbytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007960
7961 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
Michal Hocko5f127332013-07-08 16:00:40 -07007962 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007963
Michal Hocko5f127332013-07-08 16:00:40 -07007964 if (new_min_free_kbytes > user_min_free_kbytes) {
7965 min_free_kbytes = new_min_free_kbytes;
7966 if (min_free_kbytes < 128)
7967 min_free_kbytes = 128;
Joel Savitzee8eb9a2020-04-01 21:09:44 -07007968 if (min_free_kbytes > 262144)
7969 min_free_kbytes = 262144;
Michal Hocko5f127332013-07-08 16:00:40 -07007970 } else {
7971 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
7972 new_min_free_kbytes, user_min_free_kbytes);
7973 }
Minchan Kimbc75d332009-06-16 15:32:48 -07007974 setup_per_zone_wmarks();
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -07007975 refresh_zone_stat_thresholds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007976 setup_per_zone_lowmem_reserve();
Joonsoo Kim6423aa82016-08-10 16:27:49 -07007977
7978#ifdef CONFIG_NUMA
7979 setup_min_unmapped_ratio();
7980 setup_min_slab_ratio();
7981#endif
7982
Vijay Balakrishna4aab2be2020-10-10 23:16:40 -07007983 khugepaged_min_free_kbytes_update();
7984
Linus Torvalds1da177e2005-04-16 15:20:36 -07007985 return 0;
7986}
Doug Bergere08d3fd2020-08-20 17:42:24 -07007987postcore_initcall(init_per_zone_wmark_min)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007988
7989/*
Pintu Kumarb8af2942013-09-11 14:20:34 -07007990 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
Linus Torvalds1da177e2005-04-16 15:20:36 -07007991 * that we can call two helper functions whenever min_free_kbytes
7992 * changes.
7993 */
Joe Perchescccad5b2014-06-06 14:38:09 -07007994int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02007995 void *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007996{
Han Pingtianda8c7572014-01-23 15:53:17 -08007997 int rc;
7998
7999 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
8000 if (rc)
8001 return rc;
8002
Michal Hocko5f127332013-07-08 16:00:40 -07008003 if (write) {
8004 user_min_free_kbytes = min_free_kbytes;
Minchan Kimbc75d332009-06-16 15:32:48 -07008005 setup_per_zone_wmarks();
Michal Hocko5f127332013-07-08 16:00:40 -07008006 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008007 return 0;
8008}
8009
Johannes Weiner795ae7a2016-03-17 14:19:14 -07008010int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008011 void *buffer, size_t *length, loff_t *ppos)
Johannes Weiner795ae7a2016-03-17 14:19:14 -07008012{
8013 int rc;
8014
8015 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
8016 if (rc)
8017 return rc;
8018
8019 if (write)
8020 setup_per_zone_wmarks();
8021
8022 return 0;
8023}
8024
Christoph Lameter96146342006-07-03 00:24:13 -07008025#ifdef CONFIG_NUMA
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008026static void setup_min_unmapped_ratio(void)
Christoph Lameter96146342006-07-03 00:24:13 -07008027{
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008028 pg_data_t *pgdat;
Christoph Lameter96146342006-07-03 00:24:13 -07008029 struct zone *zone;
Christoph Lameter96146342006-07-03 00:24:13 -07008030
Mel Gormana5f5f912016-07-28 15:46:32 -07008031 for_each_online_pgdat(pgdat)
Joonsoo Kim81cbcbc2016-08-10 16:27:46 -07008032 pgdat->min_unmapped_pages = 0;
Mel Gormana5f5f912016-07-28 15:46:32 -07008033
Christoph Lameter96146342006-07-03 00:24:13 -07008034 for_each_zone(zone)
Arun KS9705bea2018-12-28 00:34:24 -08008035 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
8036 sysctl_min_unmapped_ratio) / 100;
Christoph Lameter96146342006-07-03 00:24:13 -07008037}
Christoph Lameter0ff38492006-09-25 23:31:52 -07008038
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008039
8040int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008041 void *buffer, size_t *length, loff_t *ppos)
Christoph Lameter0ff38492006-09-25 23:31:52 -07008042{
Christoph Lameter0ff38492006-09-25 23:31:52 -07008043 int rc;
8044
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008045 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
Christoph Lameter0ff38492006-09-25 23:31:52 -07008046 if (rc)
8047 return rc;
8048
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008049 setup_min_unmapped_ratio();
8050
8051 return 0;
8052}
8053
8054static void setup_min_slab_ratio(void)
8055{
8056 pg_data_t *pgdat;
8057 struct zone *zone;
8058
Mel Gormana5f5f912016-07-28 15:46:32 -07008059 for_each_online_pgdat(pgdat)
8060 pgdat->min_slab_pages = 0;
8061
Christoph Lameter0ff38492006-09-25 23:31:52 -07008062 for_each_zone(zone)
Arun KS9705bea2018-12-28 00:34:24 -08008063 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
8064 sysctl_min_slab_ratio) / 100;
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008065}
8066
8067int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008068 void *buffer, size_t *length, loff_t *ppos)
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008069{
8070 int rc;
8071
8072 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
8073 if (rc)
8074 return rc;
8075
8076 setup_min_slab_ratio();
8077
Christoph Lameter0ff38492006-09-25 23:31:52 -07008078 return 0;
8079}
Christoph Lameter96146342006-07-03 00:24:13 -07008080#endif
8081
Linus Torvalds1da177e2005-04-16 15:20:36 -07008082/*
8083 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
8084 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
8085 * whenever sysctl_lowmem_reserve_ratio changes.
8086 *
8087 * The reserve ratio obviously has absolutely no relation with the
Mel Gorman41858962009-06-16 15:32:12 -07008088 * minimum watermarks. The lowmem reserve ratio can only make sense
Linus Torvalds1da177e2005-04-16 15:20:36 -07008089 * if in function of the boot time zone sizes.
8090 */
Joe Perchescccad5b2014-06-06 14:38:09 -07008091int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008092 void *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008093{
Baoquan He86aaf252020-06-03 15:58:48 -07008094 int i;
8095
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008096 proc_dointvec_minmax(table, write, buffer, length, ppos);
Baoquan He86aaf252020-06-03 15:58:48 -07008097
8098 for (i = 0; i < MAX_NR_ZONES; i++) {
8099 if (sysctl_lowmem_reserve_ratio[i] < 1)
8100 sysctl_lowmem_reserve_ratio[i] = 0;
8101 }
8102
Linus Torvalds1da177e2005-04-16 15:20:36 -07008103 setup_per_zone_lowmem_reserve();
8104 return 0;
8105}
8106
Mel Gormancb1ef532019-11-30 17:55:11 -08008107static void __zone_pcp_update(struct zone *zone)
8108{
8109 unsigned int cpu;
8110
8111 for_each_possible_cpu(cpu)
8112 pageset_set_high_and_batch(zone,
8113 per_cpu_ptr(zone->pageset, cpu));
8114}
8115
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008116/*
8117 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
Pintu Kumarb8af2942013-09-11 14:20:34 -07008118 * cpu. It is the fraction of total pages in each zone that a hot per cpu
8119 * pagelist can have before it gets flushed back to buddy allocator.
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008120 */
Joe Perchescccad5b2014-06-06 14:38:09 -07008121int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008122 void *buffer, size_t *length, loff_t *ppos)
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008123{
8124 struct zone *zone;
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008125 int old_percpu_pagelist_fraction;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008126 int ret;
8127
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008128 mutex_lock(&pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008129 old_percpu_pagelist_fraction = percpu_pagelist_fraction;
8130
8131 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
8132 if (!write || ret < 0)
8133 goto out;
8134
8135 /* Sanity checking to avoid pcp imbalance */
8136 if (percpu_pagelist_fraction &&
8137 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
8138 percpu_pagelist_fraction = old_percpu_pagelist_fraction;
8139 ret = -EINVAL;
8140 goto out;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008141 }
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008142
8143 /* No change? */
8144 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
8145 goto out;
8146
Mel Gormancb1ef532019-11-30 17:55:11 -08008147 for_each_populated_zone(zone)
8148 __zone_pcp_update(zone);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008149out:
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008150 mutex_unlock(&pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008151 return ret;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008152}
8153
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07008154#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
8155/*
8156 * Returns the number of pages that arch has reserved but
8157 * is not known to alloc_large_system_hash().
8158 */
8159static unsigned long __init arch_reserved_kernel_pages(void)
8160{
8161 return 0;
8162}
8163#endif
8164
Linus Torvalds1da177e2005-04-16 15:20:36 -07008165/*
Pavel Tatashin90172172017-07-06 15:39:14 -07008166 * Adaptive scale is meant to reduce sizes of hash tables on large memory
8167 * machines. As memory size is increased the scale is also increased but at
8168 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
8169 * quadruples the scale is increased by one, which means the size of hash table
8170 * only doubles, instead of quadrupling as well.
8171 * Because 32-bit systems cannot have large physical memory, where this scaling
8172 * makes sense, it is disabled on such platforms.
8173 */
8174#if __BITS_PER_LONG > 32
8175#define ADAPT_SCALE_BASE (64ul << 30)
8176#define ADAPT_SCALE_SHIFT 2
8177#define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
8178#endif
8179
8180/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07008181 * allocate a large system hash table from bootmem
8182 * - it is assumed that the hash table must contain an exact power-of-2
8183 * quantity of entries
8184 * - limit is the number of hash buckets, not the total allocation size
8185 */
8186void *__init alloc_large_system_hash(const char *tablename,
8187 unsigned long bucketsize,
8188 unsigned long numentries,
8189 int scale,
8190 int flags,
8191 unsigned int *_hash_shift,
8192 unsigned int *_hash_mask,
Tim Bird31fe62b2012-05-23 13:33:35 +00008193 unsigned long low_limit,
8194 unsigned long high_limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008195{
Tim Bird31fe62b2012-05-23 13:33:35 +00008196 unsigned long long max = high_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008197 unsigned long log2qty, size;
8198 void *table = NULL;
Pavel Tatashin3749a8f2017-07-06 15:39:08 -07008199 gfp_t gfp_flags;
Nicholas Pigginec114082019-07-11 20:59:09 -07008200 bool virt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008201
8202 /* allow the kernel cmdline to have a say */
8203 if (!numentries) {
8204 /* round applicable memory size up to nearest megabyte */
Andrew Morton04903662006-12-06 20:37:33 -08008205 numentries = nr_kernel_pages;
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07008206 numentries -= arch_reserved_kernel_pages();
Jerry Zhoua7e83312013-09-11 14:20:26 -07008207
8208 /* It isn't necessary when PAGE_SIZE >= 1MB */
8209 if (PAGE_SHIFT < 20)
8210 numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008211
Pavel Tatashin90172172017-07-06 15:39:14 -07008212#if __BITS_PER_LONG > 32
8213 if (!high_limit) {
8214 unsigned long adapt;
8215
8216 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
8217 adapt <<= ADAPT_SCALE_SHIFT)
8218 scale++;
8219 }
8220#endif
8221
Linus Torvalds1da177e2005-04-16 15:20:36 -07008222 /* limit to 1 bucket per 2^scale bytes of low memory */
8223 if (scale > PAGE_SHIFT)
8224 numentries >>= (scale - PAGE_SHIFT);
8225 else
8226 numentries <<= (PAGE_SHIFT - scale);
Paul Mundt9ab37b82007-01-05 16:36:30 -08008227
8228 /* Make sure we've got at least a 0-order allocation.. */
Jan Beulich2c85f512009-09-21 17:03:07 -07008229 if (unlikely(flags & HASH_SMALL)) {
8230 /* Makes no sense without HASH_EARLY */
8231 WARN_ON(!(flags & HASH_EARLY));
8232 if (!(numentries >> *_hash_shift)) {
8233 numentries = 1UL << *_hash_shift;
8234 BUG_ON(!numentries);
8235 }
8236 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
Paul Mundt9ab37b82007-01-05 16:36:30 -08008237 numentries = PAGE_SIZE / bucketsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008238 }
John Hawkes6e692ed2006-03-25 03:08:02 -08008239 numentries = roundup_pow_of_two(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008240
8241 /* limit allocation size to 1/16 total memory by default */
8242 if (max == 0) {
8243 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
8244 do_div(max, bucketsize);
8245 }
Dimitri Sivanich074b8512012-02-08 12:39:07 -08008246 max = min(max, 0x80000000ULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008247
Tim Bird31fe62b2012-05-23 13:33:35 +00008248 if (numentries < low_limit)
8249 numentries = low_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008250 if (numentries > max)
8251 numentries = max;
8252
David Howellsf0d1b0b2006-12-08 02:37:49 -08008253 log2qty = ilog2(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008254
Pavel Tatashin3749a8f2017-07-06 15:39:08 -07008255 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008256 do {
Nicholas Pigginec114082019-07-11 20:59:09 -07008257 virt = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008258 size = bucketsize << log2qty;
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008259 if (flags & HASH_EARLY) {
8260 if (flags & HASH_ZERO)
Mike Rapoport26fb3da2019-03-11 23:30:42 -07008261 table = memblock_alloc(size, SMP_CACHE_BYTES);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008262 else
Mike Rapoport7e1c4e22018-10-30 15:09:57 -07008263 table = memblock_alloc_raw(size,
8264 SMP_CACHE_BYTES);
Nicholas Pigginec114082019-07-11 20:59:09 -07008265 } else if (get_order(size) >= MAX_ORDER || hashdist) {
Christoph Hellwig88dca4c2020-06-01 21:51:40 -07008266 table = __vmalloc(size, gfp_flags);
Nicholas Pigginec114082019-07-11 20:59:09 -07008267 virt = true;
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008268 } else {
Eric Dumazet1037b832007-07-15 23:38:05 -07008269 /*
8270 * If bucketsize is not a power-of-two, we may free
Mel Gormana1dd2682009-06-16 15:32:19 -07008271 * some pages at the end of hash table which
8272 * alloc_pages_exact() automatically does
Eric Dumazet1037b832007-07-15 23:38:05 -07008273 */
Nicholas Pigginec114082019-07-11 20:59:09 -07008274 table = alloc_pages_exact(size, gfp_flags);
8275 kmemleak_alloc(table, size, 1, gfp_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008276 }
8277 } while (!table && size > PAGE_SIZE && --log2qty);
8278
8279 if (!table)
8280 panic("Failed to allocate %s hash table\n", tablename);
8281
Nicholas Pigginec114082019-07-11 20:59:09 -07008282 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
8283 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
8284 virt ? "vmalloc" : "linear");
Linus Torvalds1da177e2005-04-16 15:20:36 -07008285
8286 if (_hash_shift)
8287 *_hash_shift = log2qty;
8288 if (_hash_mask)
8289 *_hash_mask = (1 << log2qty) - 1;
8290
8291 return table;
8292}
KAMEZAWA Hiroyukia117e662006-03-27 01:15:25 -08008293
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -07008294/*
Minchan Kim80934512012-07-31 16:43:01 -07008295 * This function checks whether pageblock includes unmovable pages or not.
Minchan Kim80934512012-07-31 16:43:01 -07008296 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07008297 * PageLRU check without isolation or lru_lock could race so that
Yisheng Xie0efadf42017-02-24 14:57:39 -08008298 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
8299 * check without lock_page also may miss some movable non-lru pages at
8300 * race condition. So you can't expect this function should be exact.
Qian Cai4a55c042020-01-30 22:14:57 -08008301 *
8302 * Returns a page without holding a reference. If the caller wants to
Randy Dunlap047b9962020-08-11 18:33:14 -07008303 * dereference that page (e.g., dumping), it has to make sure that it
Qian Cai4a55c042020-01-30 22:14:57 -08008304 * cannot get removed (e.g., via memory unplug) concurrently.
8305 *
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -07008306 */
Qian Cai4a55c042020-01-30 22:14:57 -08008307struct page *has_unmovable_pages(struct zone *zone, struct page *page,
8308 int migratetype, int flags)
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008309{
Qian Cai1a9f21912019-04-18 17:50:30 -07008310 unsigned long iter = 0;
8311 unsigned long pfn = page_to_pfn(page);
Li Xinhai6a654e32020-10-13 16:55:39 -07008312 unsigned long offset = pfn % pageblock_nr_pages;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01008313
Qian Cai1a9f21912019-04-18 17:50:30 -07008314 if (is_migrate_cma_page(page)) {
8315 /*
8316 * CMA allocations (alloc_contig_range) really need to mark
8317 * isolate CMA pageblocks even when they are not movable in fact
8318 * so consider them movable here.
8319 */
8320 if (is_migrate_cma(migratetype))
Qian Cai4a55c042020-01-30 22:14:57 -08008321 return NULL;
Michal Hocko4da2ce22017-11-15 17:33:26 -08008322
Qian Cai3d680bd2020-01-30 22:15:01 -08008323 return page;
Qian Cai1a9f21912019-04-18 17:50:30 -07008324 }
8325
Li Xinhai6a654e32020-10-13 16:55:39 -07008326 for (; iter < pageblock_nr_pages - offset; iter++) {
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008327 if (!pfn_valid_within(pfn + iter))
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008328 continue;
Namhyung Kim29723fc2011-02-25 14:44:25 -08008329
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008330 page = pfn_to_page(pfn + iter);
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008331
David Hildenbrandc9c510d2020-10-13 16:55:17 -07008332 /*
8333 * Both, bootmem allocations and memory holes are marked
8334 * PG_reserved and are unmovable. We can even have unmovable
8335 * allocations inside ZONE_MOVABLE, for example when
8336 * specifying "movablecore".
8337 */
Michal Hockod7ab3672017-11-15 17:33:30 -08008338 if (PageReserved(page))
Qian Cai3d680bd2020-01-30 22:15:01 -08008339 return page;
Michal Hockod7ab3672017-11-15 17:33:30 -08008340
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008341 /*
Michal Hocko9d789992018-11-16 15:08:15 -08008342 * If the zone is movable and we have ruled out all reserved
8343 * pages then it should be reasonably safe to assume the rest
8344 * is movable.
8345 */
8346 if (zone_idx(zone) == ZONE_MOVABLE)
8347 continue;
8348
8349 /*
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008350 * Hugepages are not in LRU lists, but they're movable.
Rik van Riel1da2f322020-04-01 21:10:31 -07008351 * THPs are on the LRU, but need to be counted as #small pages.
Wei Yang8bb4e7a2019-03-05 15:46:22 -08008352 * We need not scan over tail pages because we don't
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008353 * handle each tail page individually in migration.
8354 */
Rik van Riel1da2f322020-04-01 21:10:31 -07008355 if (PageHuge(page) || PageTransCompound(page)) {
Oscar Salvador17e2e7d2018-12-21 14:31:00 -08008356 struct page *head = compound_head(page);
8357 unsigned int skip_pages;
Aneesh Kumar K.V464c7ff2018-09-04 15:45:59 -07008358
Rik van Riel1da2f322020-04-01 21:10:31 -07008359 if (PageHuge(page)) {
8360 if (!hugepage_migration_supported(page_hstate(head)))
8361 return page;
8362 } else if (!PageLRU(head) && !__PageMovable(head)) {
Qian Cai3d680bd2020-01-30 22:15:01 -08008363 return page;
Rik van Riel1da2f322020-04-01 21:10:31 -07008364 }
Aneesh Kumar K.V464c7ff2018-09-04 15:45:59 -07008365
Matthew Wilcox (Oracle)d8c65462019-09-23 15:34:30 -07008366 skip_pages = compound_nr(head) - (page - head);
Oscar Salvador17e2e7d2018-12-21 14:31:00 -08008367 iter += skip_pages - 1;
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008368 continue;
8369 }
8370
Minchan Kim97d255c2012-07-31 16:42:59 -07008371 /*
8372 * We can't use page_count without pin a page
8373 * because another CPU can free compound page.
8374 * This check already skips compound tails of THP
Joonsoo Kim0139aa72016-05-19 17:10:49 -07008375 * because their page->_refcount is zero at all time.
Minchan Kim97d255c2012-07-31 16:42:59 -07008376 */
Joonsoo Kimfe896d12016-03-17 14:19:26 -07008377 if (!page_ref_count(page)) {
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008378 if (PageBuddy(page))
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008379 iter += (1 << buddy_order(page)) - 1;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008380 continue;
8381 }
Minchan Kim97d255c2012-07-31 16:42:59 -07008382
Wen Congyangb023f462012-12-11 16:00:45 -08008383 /*
8384 * The HWPoisoned page may be not in buddy system, and
8385 * page_count() is not 0.
8386 */
David Hildenbrand756d25b2019-11-30 17:54:07 -08008387 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
Wen Congyangb023f462012-12-11 16:00:45 -08008388 continue;
8389
David Hildenbrandaa218792020-05-07 16:01:30 +02008390 /*
8391 * We treat all PageOffline() pages as movable when offlining
8392 * to give drivers a chance to decrement their reference count
8393 * in MEM_GOING_OFFLINE in order to indicate that these pages
8394 * can be offlined as there are no direct references anymore.
8395 * For actually unmovable PageOffline() where the driver does
8396 * not support this, we will fail later when trying to actually
8397 * move these pages that still have a reference count > 0.
8398 * (false negatives in this function only)
8399 */
8400 if ((flags & MEMORY_OFFLINE) && PageOffline(page))
8401 continue;
8402
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008403 if (__PageMovable(page) || PageLRU(page))
Yisheng Xie0efadf42017-02-24 14:57:39 -08008404 continue;
8405
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008406 /*
Johannes Weiner6b4f7792014-12-12 16:56:13 -08008407 * If there are RECLAIMABLE pages, we need to check
8408 * it. But now, memory offline itself doesn't call
8409 * shrink_node_slabs() and it still to be fixed.
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008410 */
Qian Cai3d680bd2020-01-30 22:15:01 -08008411 return page;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008412 }
Qian Cai4a55c042020-01-30 22:14:57 -08008413 return NULL;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008414}
8415
Alexandre Ghiti8df995f2019-05-13 17:19:00 -07008416#ifdef CONFIG_CONTIG_ALLOC
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008417static unsigned long pfn_max_align_down(unsigned long pfn)
8418{
8419 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
8420 pageblock_nr_pages) - 1);
8421}
8422
8423static unsigned long pfn_max_align_up(unsigned long pfn)
8424{
8425 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
8426 pageblock_nr_pages));
8427}
8428
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008429/* [start, end) must belong to a single zone. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008430static int __alloc_contig_migrate_range(struct compact_control *cc,
8431 unsigned long start, unsigned long end)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008432{
8433 /* This function is based on compact_zone() from compaction.c. */
Maninder Singh730ec8c2020-06-03 16:01:18 -07008434 unsigned int nr_reclaimed;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008435 unsigned long pfn = start;
8436 unsigned int tries = 0;
8437 int ret = 0;
Joonsoo Kim8b94e0b2020-08-11 18:37:31 -07008438 struct migration_target_control mtc = {
8439 .nid = zone_to_nid(cc->zone),
8440 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
8441 };
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008442
Marek Szyprowskibe49a6e2012-12-12 13:51:19 -08008443 migrate_prep();
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008444
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008445 while (pfn < end || !list_empty(&cc->migratepages)) {
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008446 if (fatal_signal_pending(current)) {
8447 ret = -EINTR;
8448 break;
8449 }
8450
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008451 if (list_empty(&cc->migratepages)) {
8452 cc->nr_migratepages = 0;
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07008453 pfn = isolate_migratepages_range(cc, pfn, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008454 if (!pfn) {
8455 ret = -EINTR;
8456 break;
8457 }
8458 tries = 0;
8459 } else if (++tries == 5) {
8460 ret = ret < 0 ? ret : -EBUSY;
8461 break;
8462 }
8463
Minchan Kimbeb51ea2012-10-08 16:33:51 -07008464 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
8465 &cc->migratepages);
8466 cc->nr_migratepages -= nr_reclaimed;
Minchan Kim02c6de82012-10-08 16:31:55 -07008467
Joonsoo Kim8b94e0b2020-08-11 18:37:31 -07008468 ret = migrate_pages(&cc->migratepages, alloc_migration_target,
8469 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008470 }
Srinivas Pandruvada2a6f5122013-02-22 16:32:09 -08008471 if (ret < 0) {
8472 putback_movable_pages(&cc->migratepages);
8473 return ret;
8474 }
8475 return 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008476}
8477
8478/**
8479 * alloc_contig_range() -- tries to allocate given range of pages
8480 * @start: start PFN to allocate
8481 * @end: one-past-the-last PFN to allocate
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02008482 * @migratetype: migratetype of the underlaying pageblocks (either
8483 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
8484 * in range must have the same migratetype and it must
8485 * be either of the two.
Lucas Stachca96b622017-02-24 14:58:37 -08008486 * @gfp_mask: GFP mask to use during compaction
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008487 *
8488 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
Mike Kravetz2c7452a2018-04-05 16:25:26 -07008489 * aligned. The PFN range must belong to a single zone.
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008490 *
Mike Kravetz2c7452a2018-04-05 16:25:26 -07008491 * The first thing this routine does is attempt to MIGRATE_ISOLATE all
8492 * pageblocks in the range. Once isolated, the pageblocks should not
8493 * be modified by others.
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008494 *
Mike Rapoporta862f682019-03-05 15:48:42 -08008495 * Return: zero on success or negative error code. On success all
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008496 * pages which PFN is in [start, end) are allocated for the caller and
8497 * need to be freed with free_contig_range().
8498 */
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02008499int alloc_contig_range(unsigned long start, unsigned long end,
Lucas Stachca96b622017-02-24 14:58:37 -08008500 unsigned migratetype, gfp_t gfp_mask)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008501{
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008502 unsigned long outer_start, outer_end;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08008503 unsigned int order;
8504 int ret = 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008505
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008506 struct compact_control cc = {
8507 .nr_migratepages = 0,
8508 .order = -1,
8509 .zone = page_zone(pfn_to_page(start)),
David Rientjese0b9dae2014-06-04 16:08:28 -07008510 .mode = MIGRATE_SYNC,
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008511 .ignore_skip_hint = true,
Vlastimil Babka2583d672017-11-17 15:26:38 -08008512 .no_set_skip_hint = true,
Michal Hocko7dea19f2017-05-03 14:53:15 -07008513 .gfp_mask = current_gfp_context(gfp_mask),
Rik van Rielb06eda02020-04-01 21:10:28 -07008514 .alloc_contig = true,
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008515 };
8516 INIT_LIST_HEAD(&cc.migratepages);
8517
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008518 /*
8519 * What we do here is we mark all pageblocks in range as
8520 * MIGRATE_ISOLATE. Because pageblock and max order pages may
8521 * have different sizes, and due to the way page allocator
8522 * work, we align the range to biggest of the two pages so
8523 * that page allocator won't try to merge buddies from
8524 * different pageblocks and change MIGRATE_ISOLATE to some
8525 * other migration type.
8526 *
8527 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
8528 * migrate the pages from an unaligned range (ie. pages that
8529 * we are interested in). This will put all the pages in
8530 * range back to page allocator as MIGRATE_ISOLATE.
8531 *
8532 * When this is done, we take the pages in range from page
8533 * allocator removing them from the buddy system. This way
8534 * page allocator will never consider using them.
8535 *
8536 * This lets us mark the pageblocks back as
8537 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
8538 * aligned range but not in the unaligned, original range are
8539 * put back to page allocator so that buddy can use them.
8540 */
8541
8542 ret = start_isolate_page_range(pfn_max_align_down(start),
Michal Hockod381c542018-12-28 00:33:56 -08008543 pfn_max_align_up(end), migratetype, 0);
David Hildenbrand3fa0c7c2020-10-15 20:08:07 -07008544 if (ret)
Bob Liu86a595f2012-10-25 13:37:56 -07008545 return ret;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008546
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008547 /*
8548 * In case of -EBUSY, we'd like to know which page causes problem.
Mike Kravetz63cd4482017-11-29 16:10:01 -08008549 * So, just fall through. test_pages_isolated() has a tracepoint
8550 * which will report the busy page.
8551 *
8552 * It is possible that busy pages could become available before
8553 * the call to test_pages_isolated, and the range will actually be
8554 * allocated. So, if we fall through be sure to clear ret so that
8555 * -EBUSY is not accidentally used or returned to caller.
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008556 */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008557 ret = __alloc_contig_migrate_range(&cc, start, end);
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008558 if (ret && ret != -EBUSY)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008559 goto done;
Mike Kravetz63cd4482017-11-29 16:10:01 -08008560 ret =0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008561
8562 /*
8563 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
8564 * aligned blocks that are marked as MIGRATE_ISOLATE. What's
8565 * more, all pages in [start, end) are free in page allocator.
8566 * What we are going to do is to allocate all pages from
8567 * [start, end) (that is remove them from page allocator).
8568 *
8569 * The only problem is that pages at the beginning and at the
8570 * end of interesting range may be not aligned with pages that
8571 * page allocator holds, ie. they can be part of higher order
8572 * pages. Because of this, we reserve the bigger range and
8573 * once this is done free the pages we are not interested in.
8574 *
8575 * We don't have to hold zone->lock here because the pages are
8576 * isolated thus they won't get removed from buddy.
8577 */
8578
8579 lru_add_drain_all();
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008580
8581 order = 0;
8582 outer_start = start;
8583 while (!PageBuddy(pfn_to_page(outer_start))) {
8584 if (++order >= MAX_ORDER) {
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008585 outer_start = start;
8586 break;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008587 }
8588 outer_start &= ~0UL << order;
8589 }
8590
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008591 if (outer_start != start) {
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008592 order = buddy_order(pfn_to_page(outer_start));
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008593
8594 /*
8595 * outer_start page could be small order buddy page and
8596 * it doesn't include start page. Adjust outer_start
8597 * in this case to report failed page properly
8598 * on tracepoint in test_pages_isolated()
8599 */
8600 if (outer_start + (1UL << order) <= start)
8601 outer_start = start;
8602 }
8603
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008604 /* Make sure the range is really isolated. */
David Hildenbrand756d25b2019-11-30 17:54:07 -08008605 if (test_pages_isolated(outer_start, end, 0)) {
Jonathan Toppins75dddef2017-08-10 15:23:35 -07008606 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
Michal Nazarewiczdae803e2014-11-13 15:19:27 -08008607 __func__, outer_start, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008608 ret = -EBUSY;
8609 goto done;
8610 }
8611
Marek Szyprowski49f223a2012-01-25 12:49:24 +01008612 /* Grab isolated pages from freelists. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008613 outer_end = isolate_freepages_range(&cc, outer_start, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008614 if (!outer_end) {
8615 ret = -EBUSY;
8616 goto done;
8617 }
8618
8619 /* Free head and tail (if any) */
8620 if (start != outer_start)
8621 free_contig_range(outer_start, start - outer_start);
8622 if (end != outer_end)
8623 free_contig_range(end, outer_end - end);
8624
8625done:
8626 undo_isolate_page_range(pfn_max_align_down(start),
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02008627 pfn_max_align_up(end), migratetype);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008628 return ret;
8629}
David Hildenbrand255f5982020-05-07 16:01:29 +02008630EXPORT_SYMBOL(alloc_contig_range);
Anshuman Khandual5e27a2d2019-11-30 17:55:06 -08008631
8632static int __alloc_contig_pages(unsigned long start_pfn,
8633 unsigned long nr_pages, gfp_t gfp_mask)
8634{
8635 unsigned long end_pfn = start_pfn + nr_pages;
8636
8637 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
8638 gfp_mask);
8639}
8640
8641static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
8642 unsigned long nr_pages)
8643{
8644 unsigned long i, end_pfn = start_pfn + nr_pages;
8645 struct page *page;
8646
8647 for (i = start_pfn; i < end_pfn; i++) {
8648 page = pfn_to_online_page(i);
8649 if (!page)
8650 return false;
8651
8652 if (page_zone(page) != z)
8653 return false;
8654
8655 if (PageReserved(page))
8656 return false;
8657
8658 if (page_count(page) > 0)
8659 return false;
8660
8661 if (PageHuge(page))
8662 return false;
8663 }
8664 return true;
8665}
8666
8667static bool zone_spans_last_pfn(const struct zone *zone,
8668 unsigned long start_pfn, unsigned long nr_pages)
8669{
8670 unsigned long last_pfn = start_pfn + nr_pages - 1;
8671
8672 return zone_spans_pfn(zone, last_pfn);
8673}
8674
8675/**
8676 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
8677 * @nr_pages: Number of contiguous pages to allocate
8678 * @gfp_mask: GFP mask to limit search and used during compaction
8679 * @nid: Target node
8680 * @nodemask: Mask for other possible nodes
8681 *
8682 * This routine is a wrapper around alloc_contig_range(). It scans over zones
8683 * on an applicable zonelist to find a contiguous pfn range which can then be
8684 * tried for allocation with alloc_contig_range(). This routine is intended
8685 * for allocation requests which can not be fulfilled with the buddy allocator.
8686 *
8687 * The allocated memory is always aligned to a page boundary. If nr_pages is a
8688 * power of two then the alignment is guaranteed to be to the given nr_pages
8689 * (e.g. 1GB request would be aligned to 1GB).
8690 *
8691 * Allocated pages can be freed with free_contig_range() or by manually calling
8692 * __free_page() on each allocated page.
8693 *
8694 * Return: pointer to contiguous pages on success, or NULL if not successful.
8695 */
8696struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
8697 int nid, nodemask_t *nodemask)
8698{
8699 unsigned long ret, pfn, flags;
8700 struct zonelist *zonelist;
8701 struct zone *zone;
8702 struct zoneref *z;
8703
8704 zonelist = node_zonelist(nid, gfp_mask);
8705 for_each_zone_zonelist_nodemask(zone, z, zonelist,
8706 gfp_zone(gfp_mask), nodemask) {
8707 spin_lock_irqsave(&zone->lock, flags);
8708
8709 pfn = ALIGN(zone->zone_start_pfn, nr_pages);
8710 while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
8711 if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
8712 /*
8713 * We release the zone lock here because
8714 * alloc_contig_range() will also lock the zone
8715 * at some point. If there's an allocation
8716 * spinning on this lock, it may win the race
8717 * and cause alloc_contig_range() to fail...
8718 */
8719 spin_unlock_irqrestore(&zone->lock, flags);
8720 ret = __alloc_contig_pages(pfn, nr_pages,
8721 gfp_mask);
8722 if (!ret)
8723 return pfn_to_page(pfn);
8724 spin_lock_irqsave(&zone->lock, flags);
8725 }
8726 pfn += nr_pages;
8727 }
8728 spin_unlock_irqrestore(&zone->lock, flags);
8729 }
8730 return NULL;
8731}
Alexandre Ghiti4eb07162019-05-13 17:19:04 -07008732#endif /* CONFIG_CONTIG_ALLOC */
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008733
Alexandre Ghiti4eb07162019-05-13 17:19:04 -07008734void free_contig_range(unsigned long pfn, unsigned int nr_pages)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008735{
Marek Szyprowskibcc2b022012-12-20 15:05:18 -08008736 unsigned int count = 0;
8737
8738 for (; nr_pages--; pfn++) {
8739 struct page *page = pfn_to_page(pfn);
8740
8741 count += page_count(page) != 1;
8742 __free_page(page);
8743 }
8744 WARN(count != 0, "%d pages are still in use!\n", count);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008745}
David Hildenbrand255f5982020-05-07 16:01:29 +02008746EXPORT_SYMBOL(free_contig_range);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008747
Cody P Schafer0a647f32013-07-03 15:01:33 -07008748/*
8749 * The zone indicated has a new number of managed_pages; batch sizes and percpu
8750 * page high values need to be recalulated.
8751 */
Jiang Liu4ed7e022012-07-31 16:43:35 -07008752void __meminit zone_pcp_update(struct zone *zone)
8753{
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008754 mutex_lock(&pcp_batch_high_lock);
Mel Gormancb1ef532019-11-30 17:55:11 -08008755 __zone_pcp_update(zone);
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008756 mutex_unlock(&pcp_batch_high_lock);
Jiang Liu4ed7e022012-07-31 16:43:35 -07008757}
Jiang Liu4ed7e022012-07-31 16:43:35 -07008758
Jiang Liu340175b2012-07-31 16:43:32 -07008759void zone_pcp_reset(struct zone *zone)
8760{
8761 unsigned long flags;
Minchan Kim5a883812012-10-08 16:33:39 -07008762 int cpu;
8763 struct per_cpu_pageset *pset;
Jiang Liu340175b2012-07-31 16:43:32 -07008764
8765 /* avoid races with drain_pages() */
8766 local_irq_save(flags);
8767 if (zone->pageset != &boot_pageset) {
Minchan Kim5a883812012-10-08 16:33:39 -07008768 for_each_online_cpu(cpu) {
8769 pset = per_cpu_ptr(zone->pageset, cpu);
8770 drain_zonestat(zone, pset);
8771 }
Jiang Liu340175b2012-07-31 16:43:32 -07008772 free_percpu(zone->pageset);
8773 zone->pageset = &boot_pageset;
8774 }
8775 local_irq_restore(flags);
8776}
8777
Wen Congyang6dcd73d2012-12-11 16:01:01 -08008778#ifdef CONFIG_MEMORY_HOTREMOVE
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008779/*
David Hildenbrand257bea72020-10-15 20:07:59 -07008780 * All pages in the range must be in a single zone, must not contain holes,
8781 * must span full sections, and must be isolated before calling this function.
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008782 */
David Hildenbrand257bea72020-10-15 20:07:59 -07008783void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008784{
David Hildenbrand257bea72020-10-15 20:07:59 -07008785 unsigned long pfn = start_pfn;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008786 struct page *page;
8787 struct zone *zone;
David Hildenbrand0ee5f4f2019-11-30 17:54:03 -08008788 unsigned int order;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008789 unsigned long flags;
Michal Hocko5557c762019-05-13 17:21:24 -07008790
Michal Hocko2d070ea2017-07-06 15:37:56 -07008791 offline_mem_sections(pfn, end_pfn);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008792 zone = page_zone(pfn_to_page(pfn));
8793 spin_lock_irqsave(&zone->lock, flags);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008794 while (pfn < end_pfn) {
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008795 page = pfn_to_page(pfn);
Wen Congyangb023f462012-12-11 16:00:45 -08008796 /*
8797 * The HWPoisoned page may be not in buddy system, and
8798 * page_count() is not 0.
8799 */
8800 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
8801 pfn++;
Wen Congyangb023f462012-12-11 16:00:45 -08008802 continue;
8803 }
David Hildenbrandaa218792020-05-07 16:01:30 +02008804 /*
8805 * At this point all remaining PageOffline() pages have a
8806 * reference count of 0 and can simply be skipped.
8807 */
8808 if (PageOffline(page)) {
8809 BUG_ON(page_count(page));
8810 BUG_ON(PageBuddy(page));
8811 pfn++;
David Hildenbrandaa218792020-05-07 16:01:30 +02008812 continue;
8813 }
Wen Congyangb023f462012-12-11 16:00:45 -08008814
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008815 BUG_ON(page_count(page));
8816 BUG_ON(!PageBuddy(page));
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008817 order = buddy_order(page);
Alexander Duyck6ab01362020-04-06 20:04:49 -07008818 del_page_from_free_list(page, zone, order);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008819 pfn += (1 << order);
8820 }
8821 spin_unlock_irqrestore(&zone->lock, flags);
8822}
8823#endif
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008824
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008825bool is_free_buddy_page(struct page *page)
8826{
8827 struct zone *zone = page_zone(page);
8828 unsigned long pfn = page_to_pfn(page);
8829 unsigned long flags;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07008830 unsigned int order;
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008831
8832 spin_lock_irqsave(&zone->lock, flags);
8833 for (order = 0; order < MAX_ORDER; order++) {
8834 struct page *page_head = page - (pfn & ((1 << order) - 1));
8835
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008836 if (PageBuddy(page_head) && buddy_order(page_head) >= order)
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008837 break;
8838 }
8839 spin_unlock_irqrestore(&zone->lock, flags);
8840
8841 return order < MAX_ORDER;
8842}
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008843
8844#ifdef CONFIG_MEMORY_FAILURE
8845/*
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008846 * Break down a higher-order page in sub-pages, and keep our target out of
8847 * buddy allocator.
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008848 */
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008849static void break_down_buddy_pages(struct zone *zone, struct page *page,
8850 struct page *target, int low, int high,
8851 int migratetype)
8852{
8853 unsigned long size = 1 << high;
8854 struct page *current_buddy, *next_page;
8855
8856 while (high > low) {
8857 high--;
8858 size >>= 1;
8859
8860 if (target >= &page[size]) {
8861 next_page = page + size;
8862 current_buddy = page;
8863 } else {
8864 next_page = page;
8865 current_buddy = page + size;
8866 }
8867
8868 if (set_page_guard(zone, current_buddy, high, migratetype))
8869 continue;
8870
8871 if (current_buddy != target) {
8872 add_to_free_list(current_buddy, zone, high, migratetype);
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008873 set_buddy_order(current_buddy, high);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008874 page = next_page;
8875 }
8876 }
8877}
8878
8879/*
8880 * Take a page that will be marked as poisoned off the buddy allocator.
8881 */
8882bool take_page_off_buddy(struct page *page)
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008883{
8884 struct zone *zone = page_zone(page);
8885 unsigned long pfn = page_to_pfn(page);
8886 unsigned long flags;
8887 unsigned int order;
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008888 bool ret = false;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008889
8890 spin_lock_irqsave(&zone->lock, flags);
8891 for (order = 0; order < MAX_ORDER; order++) {
8892 struct page *page_head = page - (pfn & ((1 << order) - 1));
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008893 int page_order = buddy_order(page_head);
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008894
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008895 if (PageBuddy(page_head) && page_order >= order) {
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008896 unsigned long pfn_head = page_to_pfn(page_head);
8897 int migratetype = get_pfnblock_migratetype(page_head,
8898 pfn_head);
8899
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008900 del_page_from_free_list(page_head, zone, page_order);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008901 break_down_buddy_pages(zone, page_head, page, 0,
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008902 page_order, migratetype);
Ding Hui68dcd322021-06-04 20:01:21 -07008903 if (!is_migrate_isolate(migratetype))
8904 __mod_zone_freepage_state(zone, -1, migratetype);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008905 ret = true;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008906 break;
8907 }
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008908 if (page_count(page_head) > 0)
8909 break;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008910 }
8911 spin_unlock_irqrestore(&zone->lock, flags);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008912 return ret;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008913}
8914#endif
Baoquan Hed2e57242022-01-14 14:07:37 -08008915
8916#ifdef CONFIG_ZONE_DMA
8917bool has_managed_dma(void)
8918{
8919 struct pglist_data *pgdat;
8920
8921 for_each_online_pgdat(pgdat) {
8922 struct zone *zone = &pgdat->node_zones[ZONE_DMA];
8923
8924 if (managed_zone(zone))
8925 return true;
8926 }
8927 return false;
8928}
8929#endif /* CONFIG_ZONE_DMA */