blob: 3589febc6d31928f850ebe5a4015ddc40e0469f3 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/mm/page_alloc.c
4 *
5 * Manages the free list, the system allocates free pages here.
6 * Note that kmalloc() lives in slab.c
7 *
8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
9 * Swap reorganised 29.12.95, Stephen Tweedie
10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
16 */
17
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/stddef.h>
19#include <linux/mm.h>
Arun KSca79b0c2018-12-28 00:34:29 -080020#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/swap.h>
Naoya Horiguchibf181c52022-01-14 14:09:09 -080022#include <linux/swapops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/interrupt.h>
24#include <linux/pagemap.h>
KOSAKI Motohiro10ed2732008-03-04 14:28:32 -080025#include <linux/jiffies.h>
Yinghai Luedbe7d22010-08-25 13:39:16 -070026#include <linux/memblock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/compiler.h>
Randy Dunlap9f158332005-09-13 01:25:16 -070028#include <linux/kernel.h>
Andrey Ryabininb8c73fc2015-02-13 14:39:28 -080029#include <linux/kasan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/module.h>
31#include <linux/suspend.h>
32#include <linux/pagevec.h>
33#include <linux/blkdev.h>
34#include <linux/slab.h>
Dave Hansena238ab52011-05-24 17:12:16 -070035#include <linux/ratelimit.h>
David Rientjes5a3135c22007-10-16 23:25:53 -070036#include <linux/oom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037#include <linux/topology.h>
38#include <linux/sysctl.h>
39#include <linux/cpu.h>
40#include <linux/cpuset.h>
Dave Hansenbdc8cb92005-10-29 18:16:53 -070041#include <linux/memory_hotplug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <linux/nodemask.h>
43#include <linux/vmalloc.h>
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -070044#include <linux/vmstat.h>
Christoph Lameter4be38e32006-01-06 00:11:17 -080045#include <linux/mempolicy.h>
Dan Williams4b94ffd2016-01-15 16:56:22 -080046#include <linux/memremap.h>
Yasunori Goto68113782006-06-23 02:03:11 -070047#include <linux/stop_machine.h>
Dan Williams97500a42019-05-14 15:41:35 -070048#include <linux/random.h>
Mel Gormanc7132162006-09-27 01:49:43 -070049#include <linux/sort.h>
50#include <linux/pfn.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070051#include <linux/backing-dev.h>
Akinobu Mita933e3122006-12-08 02:39:45 -080052#include <linux/fault-inject.h>
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -070053#include <linux/page-isolation.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070054#include <linux/debugobjects.h>
Catalin Marinasdbb1f812009-06-11 13:23:19 +010055#include <linux/kmemleak.h>
Mel Gorman56de7262010-05-24 14:32:30 -070056#include <linux/compaction.h>
Mel Gorman0d3d0622009-09-21 17:02:44 -070057#include <trace/events/kmem.h>
Michal Hockod379f012017-02-22 15:42:00 -080058#include <trace/events/oom.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070059#include <linux/prefetch.h>
Lisa Du6e543d52013-09-11 14:22:36 -070060#include <linux/mm_inline.h>
Daniel Vetterf920e412020-12-14 19:08:30 -080061#include <linux/mmu_notifier.h>
Michal Nazarewicz041d3a82011-12-29 13:09:50 +010062#include <linux/migrate.h>
David Rientjes949f7ec2013-04-29 15:07:48 -070063#include <linux/hugetlb.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060064#include <linux/sched/rt.h>
Ingo Molnar5b3cc152017-02-02 20:43:54 +010065#include <linux/sched/mm.h>
Joonsoo Kim48c96a32014-12-12 16:56:01 -080066#include <linux/page_owner.h>
Pasha Tatashindf4e8172022-01-14 14:06:37 -080067#include <linux/page_table_check.h>
Mel Gorman0e1cc952015-06-30 14:57:27 -070068#include <linux/kthread.h>
Vladimir Davydov49491482016-07-26 15:24:24 -070069#include <linux/memcontrol.h>
Steven Rostedt (VMware)42c269c2017-03-03 16:15:39 -050070#include <linux/ftrace.h>
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +010071#include <linux/lockdep.h>
Chen Yu556b9692017-08-25 15:55:30 -070072#include <linux/nmi.h>
Johannes Weinereb414682018-10-26 15:06:27 -070073#include <linux/psi.h>
Daniel Jordane4443142020-06-03 15:59:51 -070074#include <linux/padata.h>
Vijay Balakrishna4aab2be2020-10-10 23:16:40 -070075#include <linux/khugepaged.h>
Lin Fengba8f3582020-12-14 19:11:19 -080076#include <linux/buffer_head.h>
wangyong5bf18282022-01-19 18:10:15 -080077#include <linux/delayacct.h>
Jiang Liu7ee3d4e2013-07-03 15:03:41 -070078#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <asm/tlbflush.h>
Andrew Mortonac924c62006-05-15 09:43:59 -070080#include <asm/div64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#include "internal.h"
Dan Williamse900a912019-05-14 15:41:28 -070082#include "shuffle.h"
Alexander Duyck36e66c52020-04-06 20:04:56 -070083#include "page_reporting.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070084
David Hildenbrandf04a5d52020-10-15 20:09:20 -070085/* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
86typedef int __bitwise fpi_t;
87
88/* No special request */
89#define FPI_NONE ((__force fpi_t)0)
90
91/*
92 * Skip free page reporting notification for the (possibly merged) page.
93 * This does not hinder free page reporting from grabbing the page,
94 * reporting it and marking it "reported" - it only skips notifying
95 * the free page reporting infrastructure about a newly freed page. For
96 * example, used when temporarily pulling a page from a freelist and
97 * putting it back unmodified.
98 */
99#define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0))
100
David Hildenbrand47b6a24a22020-10-15 20:09:26 -0700101/*
102 * Place the (possibly merged) page to the tail of the freelist. Will ignore
103 * page shuffling (relevant code - e.g., memory onlining - is expected to
104 * shuffle the whole zone).
105 *
106 * Note: No code should rely on this flag for correctness - it's purely
107 * to allow for optimizations when handing back either fresh pages
108 * (memory onlining) or untouched pages (page isolation, free page
109 * reporting).
110 */
111#define FPI_TO_TAIL ((__force fpi_t)BIT(1))
112
Andrey Konovalov2c335682021-04-29 22:59:52 -0700113/*
114 * Don't poison memory with KASAN (only for the tag-based modes).
115 * During boot, all non-reserved memblock memory is exposed to page_alloc.
116 * Poisoning all that memory lengthens boot time, especially on systems with
117 * large amount of RAM. This flag is used to skip that poisoning.
118 * This is only done for the tag-based KASAN modes, as those are able to
119 * detect memory corruptions with the memory tags assigned by default.
120 * All memory allocated normally after boot gets poisoned as usual.
121 */
122#define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2))
123
Cody P Schaferc8e251f2013-07-03 15:01:29 -0700124/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
125static DEFINE_MUTEX(pcp_batch_high_lock);
Mel Gorman74f44822021-06-28 19:42:24 -0700126#define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
Cody P Schaferc8e251f2013-07-03 15:01:29 -0700127
Mel Gormandbbee9d2021-06-28 19:41:41 -0700128struct pagesets {
129 local_lock_t lock;
Mel Gormandbbee9d2021-06-28 19:41:41 -0700130};
131static DEFINE_PER_CPU(struct pagesets, pagesets) = {
132 .lock = INIT_LOCAL_LOCK(lock),
133};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
Lee Schermerhorn72812012010-05-26 14:44:56 -0700135#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
136DEFINE_PER_CPU(int, numa_node);
137EXPORT_PER_CPU_SYMBOL(numa_node);
138#endif
139
Kemi Wang45180852017-11-15 17:38:22 -0800140DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
141
Lee Schermerhorn7aac7892010-05-26 14:45:00 -0700142#ifdef CONFIG_HAVE_MEMORYLESS_NODES
143/*
144 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
145 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
146 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
147 * defined in <linux/topology.h>.
148 */
149DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
150EXPORT_PER_CPU_SYMBOL(_numa_mem_);
151#endif
152
Mel Gormanbd233f52017-02-24 14:56:56 -0800153/* work_structs for global per-cpu drains */
Wei Yangd9367bd2018-12-28 00:38:58 -0800154struct pcpu_drain {
155 struct zone *zone;
156 struct work_struct work;
157};
Jason Yan8b885f52020-04-10 14:32:32 -0700158static DEFINE_MUTEX(pcpu_drain_mutex);
159static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
Mel Gormanbd233f52017-02-24 14:56:56 -0800160
Emese Revfy38addce2016-06-20 20:41:19 +0200161#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
Kees Cook58bea412016-10-19 00:08:04 +0200162volatile unsigned long latent_entropy __latent_entropy;
Emese Revfy38addce2016-06-20 20:41:19 +0200163EXPORT_SYMBOL(latent_entropy);
164#endif
165
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166/*
Christoph Lameter13808912007-10-16 01:25:27 -0700167 * Array of node states.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 */
Christoph Lameter13808912007-10-16 01:25:27 -0700169nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
170 [N_POSSIBLE] = NODE_MASK_ALL,
171 [N_ONLINE] = { { [0] = 1UL } },
172#ifndef CONFIG_NUMA
173 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
174#ifdef CONFIG_HIGHMEM
175 [N_HIGH_MEMORY] = { { [0] = 1UL } },
176#endif
Lai Jiangshan20b2f522012-12-12 13:52:00 -0800177 [N_MEMORY] = { { [0] = 1UL } },
Christoph Lameter13808912007-10-16 01:25:27 -0700178 [N_CPU] = { { [0] = 1UL } },
179#endif /* NUMA */
180};
181EXPORT_SYMBOL(node_states);
182
Arun KSca79b0c2018-12-28 00:34:29 -0800183atomic_long_t _totalram_pages __read_mostly;
184EXPORT_SYMBOL(_totalram_pages);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -0700185unsigned long totalreserve_pages __read_mostly;
Pintu Kumare48322a2014-12-18 16:17:15 -0800186unsigned long totalcma_pages __read_mostly;
Johannes Weinerab8fabd2012-01-10 15:07:42 -0800187
Mel Gorman74f44822021-06-28 19:42:24 -0700188int percpu_pagelist_high_fraction;
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +1000189gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
Kees Cook51cba1e2021-04-01 16:23:43 -0700190DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
Alexander Potapenko64713842019-07-11 20:59:19 -0700191EXPORT_SYMBOL(init_on_alloc);
192
Kees Cook51cba1e2021-04-01 16:23:43 -0700193DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
Alexander Potapenko64713842019-07-11 20:59:19 -0700194EXPORT_SYMBOL(init_on_free);
195
Vlastimil Babka04013512020-12-14 19:13:30 -0800196static bool _init_on_alloc_enabled_early __read_mostly
197 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
Alexander Potapenko64713842019-07-11 20:59:19 -0700198static int __init early_init_on_alloc(char *buf)
199{
Alexander Potapenko64713842019-07-11 20:59:19 -0700200
Vlastimil Babka04013512020-12-14 19:13:30 -0800201 return kstrtobool(buf, &_init_on_alloc_enabled_early);
Alexander Potapenko64713842019-07-11 20:59:19 -0700202}
203early_param("init_on_alloc", early_init_on_alloc);
204
Vlastimil Babka04013512020-12-14 19:13:30 -0800205static bool _init_on_free_enabled_early __read_mostly
206 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
Alexander Potapenko64713842019-07-11 20:59:19 -0700207static int __init early_init_on_free(char *buf)
208{
Vlastimil Babka04013512020-12-14 19:13:30 -0800209 return kstrtobool(buf, &_init_on_free_enabled_early);
Alexander Potapenko64713842019-07-11 20:59:19 -0700210}
211early_param("init_on_free", early_init_on_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -0700213/*
214 * A cached value of the page's pageblock's migratetype, used when the page is
215 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
216 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
217 * Also the migratetype set in the page does not necessarily match the pcplist
218 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
219 * other index - this ensures that it will be put on the correct CMA freelist.
220 */
221static inline int get_pcppage_migratetype(struct page *page)
222{
223 return page->index;
224}
225
226static inline void set_pcppage_migratetype(struct page *page, int migratetype)
227{
228 page->index = migratetype;
229}
230
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800231#ifdef CONFIG_PM_SLEEP
232/*
233 * The following functions are used by the suspend/hibernate code to temporarily
234 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
235 * while devices are suspended. To avoid races with the suspend/hibernate code,
Pingfan Liu55f25032018-07-31 16:51:32 +0800236 * they should always be called with system_transition_mutex held
237 * (gfp_allowed_mask also should only be modified with system_transition_mutex
238 * held, unless the suspend/hibernate code is guaranteed not to run in parallel
239 * with that modification).
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800240 */
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100241
242static gfp_t saved_gfp_mask;
243
244void pm_restore_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800245{
Pingfan Liu55f25032018-07-31 16:51:32 +0800246 WARN_ON(!mutex_is_locked(&system_transition_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100247 if (saved_gfp_mask) {
248 gfp_allowed_mask = saved_gfp_mask;
249 saved_gfp_mask = 0;
250 }
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800251}
252
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100253void pm_restrict_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800254{
Pingfan Liu55f25032018-07-31 16:51:32 +0800255 WARN_ON(!mutex_is_locked(&system_transition_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100256 WARN_ON(saved_gfp_mask);
257 saved_gfp_mask = gfp_allowed_mask;
Mel Gormand0164ad2015-11-06 16:28:21 -0800258 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800259}
Mel Gormanf90ac392012-01-10 15:07:15 -0800260
261bool pm_suspended_storage(void)
262{
Mel Gormand0164ad2015-11-06 16:28:21 -0800263 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
Mel Gormanf90ac392012-01-10 15:07:15 -0800264 return false;
265 return true;
266}
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800267#endif /* CONFIG_PM_SLEEP */
268
Mel Gormand9c23402007-10-16 01:26:01 -0700269#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800270unsigned int pageblock_order __read_mostly;
Mel Gormand9c23402007-10-16 01:26:01 -0700271#endif
272
David Hildenbrand7fef4312020-10-15 20:09:35 -0700273static void __free_pages_ok(struct page *page, unsigned int order,
274 fpi_t fpi_flags);
David Howellsa226f6c2006-01-06 00:11:08 -0800275
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276/*
277 * results with 256, 32 in the lowmem_reserve sysctl:
278 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
279 * 1G machine -> (16M dma, 784M normal, 224M high)
280 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
281 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
Yaowei Bai84109e12015-02-12 15:00:22 -0800282 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
Andi Kleena2f1b422005-11-05 17:25:53 +0100283 *
284 * TBD: should special case ZONE_DMA32 machines here - in those we normally
285 * don't need any ZONE_NORMAL reservation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 */
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700287int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800288#ifdef CONFIG_ZONE_DMA
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700289 [ZONE_DMA] = 256,
Christoph Lameter4b51d662007-02-10 01:43:10 -0800290#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700291#ifdef CONFIG_ZONE_DMA32
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700292 [ZONE_DMA32] = 256,
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700293#endif
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700294 [ZONE_NORMAL] = 32,
Christoph Lametere53ef382006-09-25 23:31:14 -0700295#ifdef CONFIG_HIGHMEM
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700296 [ZONE_HIGHMEM] = 0,
Christoph Lametere53ef382006-09-25 23:31:14 -0700297#endif
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700298 [ZONE_MOVABLE] = 0,
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700299};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300
Helge Deller15ad7cd2006-12-06 20:40:36 -0800301static char * const zone_names[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800302#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700303 "DMA",
Christoph Lameter4b51d662007-02-10 01:43:10 -0800304#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700305#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700306 "DMA32",
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700307#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700308 "Normal",
Christoph Lametere53ef382006-09-25 23:31:14 -0700309#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700310 "HighMem",
Christoph Lametere53ef382006-09-25 23:31:14 -0700311#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700312 "Movable",
Dan Williams033fbae2015-08-09 15:29:06 -0400313#ifdef CONFIG_ZONE_DEVICE
314 "Device",
315#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700316};
317
Alexey Dobriyanc999fbd2018-12-28 00:35:55 -0800318const char * const migratetype_names[MIGRATE_TYPES] = {
Vlastimil Babka60f30352016-03-15 14:56:08 -0700319 "Unmovable",
320 "Movable",
321 "Reclaimable",
322 "HighAtomic",
323#ifdef CONFIG_CMA
324 "CMA",
325#endif
326#ifdef CONFIG_MEMORY_ISOLATION
327 "Isolate",
328#endif
329};
330
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700331compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
332 [NULL_COMPOUND_DTOR] = NULL,
333 [COMPOUND_PAGE_DTOR] = free_compound_page,
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800334#ifdef CONFIG_HUGETLB_PAGE
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700335 [HUGETLB_PAGE_DTOR] = free_huge_page,
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800336#endif
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800337#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700338 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800339#endif
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800340};
341
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342int min_free_kbytes = 1024;
Han Pingtian42aa83c2014-01-23 15:53:28 -0800343int user_min_free_kbytes = -1;
Mel Gorman1c308442018-12-28 00:35:52 -0800344int watermark_boost_factor __read_mostly = 15000;
Johannes Weiner795ae7a2016-03-17 14:19:14 -0700345int watermark_scale_factor = 10;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800347static unsigned long nr_kernel_pages __initdata;
348static unsigned long nr_all_pages __initdata;
349static unsigned long dma_reserve __initdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800351static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
352static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700353static unsigned long required_kernelcore __initdata;
David Rientjesa5c6d652018-04-05 16:23:09 -0700354static unsigned long required_kernelcore_percent __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700355static unsigned long required_movablecore __initdata;
David Rientjesa5c6d652018-04-05 16:23:09 -0700356static unsigned long required_movablecore_percent __initdata;
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800357static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700358static bool mirrored_kernelcore __meminitdata;
Mel Gormanc7132162006-09-27 01:49:43 -0700359
Tejun Heo0ee332c2011-12-08 10:22:09 -0800360/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
361int movable_zone;
362EXPORT_SYMBOL(movable_zone);
Mel Gormanc7132162006-09-27 01:49:43 -0700363
Miklos Szeredi418508c2007-05-23 13:57:55 -0700364#if MAX_NUMNODES > 1
Alexey Dobriyanb9726c22019-03-05 15:48:26 -0800365unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
Alexey Dobriyance0725f2019-03-05 15:48:29 -0800366unsigned int nr_online_nodes __read_mostly = 1;
Miklos Szeredi418508c2007-05-23 13:57:55 -0700367EXPORT_SYMBOL(nr_node_ids);
Christoph Lameter62bc62a2009-06-16 15:32:15 -0700368EXPORT_SYMBOL(nr_online_nodes);
Miklos Szeredi418508c2007-05-23 13:57:55 -0700369#endif
370
Mel Gorman9ef9acb2007-10-16 01:25:54 -0700371int page_group_by_mobility_disabled __read_mostly;
372
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700373#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Waiman Long3c0c12c2018-12-28 00:38:51 -0800374/*
375 * During boot we initialize deferred pages on-demand, as needed, but once
376 * page_alloc_init_late() has finished, the deferred pages are all initialized,
377 * and we can permanently disable that path.
378 */
379static DEFINE_STATIC_KEY_TRUE(deferred_pages);
380
381/*
Peter Collingbourne7a3b8352021-06-02 16:52:28 -0700382 * Calling kasan_poison_pages() only after deferred memory initialization
Waiman Long3c0c12c2018-12-28 00:38:51 -0800383 * has completed. Poisoning pages during deferred memory init will greatly
384 * lengthen the process and cause problem in large memory systems as the
385 * deferred pages initialization is done with interrupt disabled.
386 *
387 * Assuming that there will be no reference to those newly initialized
388 * pages before they are ever allocated, this should have no effect on
389 * KASAN memory tracking as the poison will be properly inserted at page
390 * allocation time. The only corner case is when pages are allocated by
391 * on-demand allocation and then freed again before the deferred pages
392 * initialization is done, but this is not likely to happen.
393 */
Peter Collingbournec275c5c2021-06-02 16:52:30 -0700394static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
Waiman Long3c0c12c2018-12-28 00:38:51 -0800395{
Peter Collingbourne7a3b8352021-06-02 16:52:28 -0700396 return static_branch_unlikely(&deferred_pages) ||
397 (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
Peter Collingbournec275c5c2021-06-02 16:52:30 -0700398 (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
399 PageSkipKASanPoison(page);
Waiman Long3c0c12c2018-12-28 00:38:51 -0800400}
401
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700402/* Returns true if the struct page for the pfn is uninitialised */
Mel Gorman0e1cc952015-06-30 14:57:27 -0700403static inline bool __meminit early_page_uninitialised(unsigned long pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700404{
Mel Gormanef70b6f2016-07-14 12:07:23 -0700405 int nid = early_pfn_to_nid(pfn);
406
407 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700408 return true;
409
410 return false;
411}
412
413/*
Pavel Tatashind3035be2018-10-26 15:09:37 -0700414 * Returns true when the remaining initialisation should be deferred until
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700415 * later in the boot cycle when it can be parallelised.
416 */
Pavel Tatashind3035be2018-10-26 15:09:37 -0700417static bool __meminit
418defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700419{
Pavel Tatashind3035be2018-10-26 15:09:37 -0700420 static unsigned long prev_end_pfn, nr_initialised;
421
422 /*
423 * prev_end_pfn static that contains the end of previous zone
424 * No need to protect because called very early in boot before smp_init.
425 */
426 if (prev_end_pfn != end_pfn) {
427 prev_end_pfn = end_pfn;
428 nr_initialised = 0;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700429 }
430
Pavel Tatashind3035be2018-10-26 15:09:37 -0700431 /* Always populate low zones for address-constrained allocations */
432 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
433 return false;
Wei Yang23b68cf2018-12-28 00:36:18 -0800434
Baoquan Hedc2da7b2020-12-29 15:14:37 -0800435 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
436 return true;
Wei Yang23b68cf2018-12-28 00:36:18 -0800437 /*
438 * We start only with one section of pages, more pages are added as
439 * needed until the rest of deferred pages are initialized.
440 */
Pavel Tatashind3035be2018-10-26 15:09:37 -0700441 nr_initialised++;
Wei Yang23b68cf2018-12-28 00:36:18 -0800442 if ((nr_initialised > PAGES_PER_SECTION) &&
Pavel Tatashind3035be2018-10-26 15:09:37 -0700443 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
444 NODE_DATA(nid)->first_deferred_pfn = pfn;
445 return true;
446 }
447 return false;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700448}
449#else
Peter Collingbournec275c5c2021-06-02 16:52:30 -0700450static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
Andrey Konovalov2c335682021-04-29 22:59:52 -0700451{
Peter Collingbourne7a3b8352021-06-02 16:52:28 -0700452 return (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
Peter Collingbournec275c5c2021-06-02 16:52:30 -0700453 (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
454 PageSkipKASanPoison(page);
Andrey Konovalov2c335682021-04-29 22:59:52 -0700455}
Waiman Long3c0c12c2018-12-28 00:38:51 -0800456
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700457static inline bool early_page_uninitialised(unsigned long pfn)
458{
459 return false;
460}
461
Pavel Tatashind3035be2018-10-26 15:09:37 -0700462static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700463{
Pavel Tatashind3035be2018-10-26 15:09:37 -0700464 return false;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700465}
466#endif
467
Mel Gorman0b423ca2016-05-19 17:14:27 -0700468/* Return a pointer to the bitmap storing bits affecting a block of pages */
Matthew Wilcox (Oracle)ca891f42021-06-28 19:41:22 -0700469static inline unsigned long *get_pageblock_bitmap(const struct page *page,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700470 unsigned long pfn)
471{
472#ifdef CONFIG_SPARSEMEM
Dan Williamsf1eca352019-07-18 15:57:57 -0700473 return section_to_usemap(__pfn_to_section(pfn));
Mel Gorman0b423ca2016-05-19 17:14:27 -0700474#else
475 return page_zone(page)->pageblock_flags;
476#endif /* CONFIG_SPARSEMEM */
477}
478
Matthew Wilcox (Oracle)ca891f42021-06-28 19:41:22 -0700479static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn)
Mel Gorman0b423ca2016-05-19 17:14:27 -0700480{
481#ifdef CONFIG_SPARSEMEM
482 pfn &= (PAGES_PER_SECTION-1);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700483#else
484 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700485#endif /* CONFIG_SPARSEMEM */
Wei Yang399b7952020-08-06 23:25:44 -0700486 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700487}
488
Wei Yang535b81e2020-08-06 23:25:51 -0700489static __always_inline
Matthew Wilcox (Oracle)ca891f42021-06-28 19:41:22 -0700490unsigned long __get_pfnblock_flags_mask(const struct page *page,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700491 unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700492 unsigned long mask)
493{
494 unsigned long *bitmap;
495 unsigned long bitidx, word_bitidx;
496 unsigned long word;
497
498 bitmap = get_pageblock_bitmap(page, pfn);
499 bitidx = pfn_to_bitidx(page, pfn);
500 word_bitidx = bitidx / BITS_PER_LONG;
501 bitidx &= (BITS_PER_LONG-1);
502
503 word = bitmap[word_bitidx];
Wei Yangd93d5ab2020-08-06 23:25:48 -0700504 return (word >> bitidx) & mask;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700505}
506
Mauro Carvalho Chehaba00cda32020-12-14 19:14:39 -0800507/**
508 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
509 * @page: The page within the block of interest
510 * @pfn: The target page frame number
511 * @mask: mask of bits that the caller is interested in
512 *
513 * Return: pageblock_bits flags
514 */
Matthew Wilcox (Oracle)ca891f42021-06-28 19:41:22 -0700515unsigned long get_pfnblock_flags_mask(const struct page *page,
516 unsigned long pfn, unsigned long mask)
Mel Gorman0b423ca2016-05-19 17:14:27 -0700517{
Wei Yang535b81e2020-08-06 23:25:51 -0700518 return __get_pfnblock_flags_mask(page, pfn, mask);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700519}
520
Matthew Wilcox (Oracle)ca891f42021-06-28 19:41:22 -0700521static __always_inline int get_pfnblock_migratetype(const struct page *page,
522 unsigned long pfn)
Mel Gorman0b423ca2016-05-19 17:14:27 -0700523{
Wei Yang535b81e2020-08-06 23:25:51 -0700524 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700525}
526
527/**
528 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
529 * @page: The page within the block of interest
530 * @flags: The flags to set
531 * @pfn: The target page frame number
Mel Gorman0b423ca2016-05-19 17:14:27 -0700532 * @mask: mask of bits that the caller is interested in
533 */
534void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
535 unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700536 unsigned long mask)
537{
538 unsigned long *bitmap;
539 unsigned long bitidx, word_bitidx;
540 unsigned long old_word, word;
541
542 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
Pingfan Liu125b8602018-12-28 00:38:43 -0800543 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
Mel Gorman0b423ca2016-05-19 17:14:27 -0700544
545 bitmap = get_pageblock_bitmap(page, pfn);
546 bitidx = pfn_to_bitidx(page, pfn);
547 word_bitidx = bitidx / BITS_PER_LONG;
548 bitidx &= (BITS_PER_LONG-1);
549
550 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
551
Wei Yangd93d5ab2020-08-06 23:25:48 -0700552 mask <<= bitidx;
553 flags <<= bitidx;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700554
555 word = READ_ONCE(bitmap[word_bitidx]);
556 for (;;) {
557 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
558 if (word == old_word)
559 break;
560 word = old_word;
561 }
562}
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700563
Minchan Kimee6f5092012-07-31 16:43:50 -0700564void set_pageblock_migratetype(struct page *page, int migratetype)
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700565{
KOSAKI Motohiro5d0f3f72013-11-12 15:08:18 -0800566 if (unlikely(page_group_by_mobility_disabled &&
567 migratetype < MIGRATE_PCPTYPES))
Mel Gorman49255c62009-06-16 15:31:58 -0700568 migratetype = MIGRATE_UNMOVABLE;
569
Wei Yangd93d5ab2020-08-06 23:25:48 -0700570 set_pfnblock_flags_mask(page, (unsigned long)migratetype,
Wei Yang535b81e2020-08-06 23:25:51 -0700571 page_to_pfn(page), MIGRATETYPE_MASK);
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700572}
573
Nick Piggin13e74442006-01-06 00:10:58 -0800574#ifdef CONFIG_DEBUG_VM
Dave Hansenc6a57e12005-10-29 18:16:52 -0700575static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576{
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700577 int ret = 0;
578 unsigned seq;
579 unsigned long pfn = page_to_pfn(page);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800580 unsigned long sp, start_pfn;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700581
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700582 do {
583 seq = zone_span_seqbegin(zone);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800584 start_pfn = zone->zone_start_pfn;
585 sp = zone->spanned_pages;
Cody P Schafer108bcc92013-02-22 16:35:23 -0800586 if (!zone_spans_pfn(zone, pfn))
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700587 ret = 1;
588 } while (zone_span_seqretry(zone, seq));
589
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800590 if (ret)
Dave Hansen613813e2014-06-04 16:07:27 -0700591 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
592 pfn, zone_to_nid(zone), zone->name,
593 start_pfn, start_pfn + sp);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800594
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700595 return ret;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700596}
597
598static int page_is_consistent(struct zone *zone, struct page *page)
599{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 if (zone != page_zone(page))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700601 return 0;
602
603 return 1;
604}
605/*
606 * Temporary debugging check for pages not lying within a given zone.
607 */
Matthias Kaehlcked73d3c9f2017-07-06 15:39:23 -0700608static int __maybe_unused bad_range(struct zone *zone, struct page *page)
Dave Hansenc6a57e12005-10-29 18:16:52 -0700609{
610 if (page_outside_zone_boundaries(zone, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 return 1;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700612 if (!page_is_consistent(zone, page))
613 return 1;
614
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 return 0;
616}
Nick Piggin13e74442006-01-06 00:10:58 -0800617#else
Matthias Kaehlcked73d3c9f2017-07-06 15:39:23 -0700618static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
Nick Piggin13e74442006-01-06 00:10:58 -0800619{
620 return 0;
621}
622#endif
623
Wei Yang82a32412020-06-03 15:58:29 -0700624static void bad_page(struct page *page, const char *reason)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625{
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800626 static unsigned long resume;
627 static unsigned long nr_shown;
628 static unsigned long nr_unshown;
629
630 /*
631 * Allow a burst of 60 reports, then keep quiet for that minute;
632 * or allow a steady drip of one report per second.
633 */
634 if (nr_shown == 60) {
635 if (time_before(jiffies, resume)) {
636 nr_unshown++;
637 goto out;
638 }
639 if (nr_unshown) {
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700640 pr_alert(
Hugh Dickins1e9e6362009-01-06 14:40:13 -0800641 "BUG: Bad page state: %lu messages suppressed\n",
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800642 nr_unshown);
643 nr_unshown = 0;
644 }
645 nr_shown = 0;
646 }
647 if (nr_shown++ == 0)
648 resume = jiffies + 60 * HZ;
649
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700650 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
Hugh Dickins3dc14742009-01-06 14:40:08 -0800651 current->comm, page_to_pfn(page));
Matthew Wilcox (Oracle)d2f07ec2021-06-28 19:41:07 -0700652 dump_page(page, reason);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700653
Dave Jones4f318882011-10-31 17:07:24 -0700654 print_modules();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 dump_stack();
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800656out:
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800657 /* Leave bad fields for debug, except PageBuddy could make trouble */
Mel Gorman22b751c2013-02-22 16:34:59 -0800658 page_mapcount_reset(page); /* remove PageBuddy */
Rusty Russell373d4d02013-01-21 17:17:39 +1030659 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660}
661
Mel Gorman44042b42021-06-28 19:43:08 -0700662static inline unsigned int order_to_pindex(int migratetype, int order)
663{
664 int base = order;
665
666#ifdef CONFIG_TRANSPARENT_HUGEPAGE
667 if (order > PAGE_ALLOC_COSTLY_ORDER) {
668 VM_BUG_ON(order != pageblock_order);
669 base = PAGE_ALLOC_COSTLY_ORDER + 1;
670 }
671#else
672 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
673#endif
674
675 return (MIGRATE_PCPTYPES * base) + migratetype;
676}
677
678static inline int pindex_to_order(unsigned int pindex)
679{
680 int order = pindex / MIGRATE_PCPTYPES;
681
682#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Miaohe Linea808b42021-11-05 13:40:02 -0700683 if (order > PAGE_ALLOC_COSTLY_ORDER)
Mel Gorman44042b42021-06-28 19:43:08 -0700684 order = pageblock_order;
Mel Gorman44042b42021-06-28 19:43:08 -0700685#else
686 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
687#endif
688
689 return order;
690}
691
692static inline bool pcp_allowed_order(unsigned int order)
693{
694 if (order <= PAGE_ALLOC_COSTLY_ORDER)
695 return true;
696#ifdef CONFIG_TRANSPARENT_HUGEPAGE
697 if (order == pageblock_order)
698 return true;
699#endif
700 return false;
701}
702
Mel Gorman21d02f82021-06-28 19:42:36 -0700703static inline void free_the_page(struct page *page, unsigned int order)
704{
Mel Gorman44042b42021-06-28 19:43:08 -0700705 if (pcp_allowed_order(order)) /* Via pcp? */
706 free_unref_page(page, order);
Mel Gorman21d02f82021-06-28 19:42:36 -0700707 else
708 __free_pages_ok(page, order, FPI_NONE);
709}
710
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711/*
712 * Higher-order pages are called "compound pages". They are structured thusly:
713 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800714 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800716 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
717 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800719 * The first tail page's ->compound_dtor holds the offset in array of compound
720 * page destructors. See compound_page_dtors.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800722 * The first tail page's ->compound_order holds the order of allocation.
Hugh Dickins41d78ba2006-02-14 13:52:58 -0800723 * This usage means that zero-order pages may not be compound.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 */
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800725
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800726void free_compound_page(struct page *page)
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800727{
Matthew Wilcox (Oracle)bbc6b702021-05-01 20:42:23 -0400728 mem_cgroup_uncharge(page_folio(page));
Mel Gorman44042b42021-06-28 19:43:08 -0700729 free_the_page(page, compound_order(page));
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800730}
731
Joao Martins5b24eee2022-01-14 14:04:15 -0800732static void prep_compound_head(struct page *page, unsigned int order)
733{
734 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
735 set_compound_order(page, order);
736 atomic_set(compound_mapcount_ptr(page), -1);
737 if (hpage_pincount_available(page))
738 atomic_set(compound_pincount_ptr(page), 0);
739}
740
741static void prep_compound_tail(struct page *head, int tail_idx)
742{
743 struct page *p = head + tail_idx;
744
745 p->mapping = TAIL_MAPPING;
746 set_compound_head(p, head);
747}
748
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800749void prep_compound_page(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750{
751 int i;
752 int nr_pages = 1 << order;
753
Christoph Lameter6d777952007-05-06 14:49:40 -0700754 __SetPageHead(page);
Joao Martins5b24eee2022-01-14 14:04:15 -0800755 for (i = 1; i < nr_pages; i++)
756 prep_compound_tail(page, i);
Matthew Wilcox (Oracle)1378a5e2020-08-14 17:30:23 -0700757
Joao Martins5b24eee2022-01-14 14:04:15 -0800758 prep_compound_head(page, order);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759}
760
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800761#ifdef CONFIG_DEBUG_PAGEALLOC
762unsigned int _debug_guardpage_minorder;
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700763
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800764bool _debug_pagealloc_enabled_early __read_mostly
765 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
766EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700767DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
Joonsoo Kim505f6d22016-03-17 14:17:56 -0700768EXPORT_SYMBOL(_debug_pagealloc_enabled);
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700769
770DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800771
Joonsoo Kim031bc572014-12-12 16:55:52 -0800772static int __init early_debug_pagealloc(char *buf)
773{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800774 return kstrtobool(buf, &_debug_pagealloc_enabled_early);
Joonsoo Kim031bc572014-12-12 16:55:52 -0800775}
776early_param("debug_pagealloc", early_debug_pagealloc);
777
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800778static int __init debug_guardpage_minorder_setup(char *buf)
779{
780 unsigned long res;
781
782 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
Joe Perches11705322016-03-17 14:19:50 -0700783 pr_err("Bad debug_guardpage_minorder value\n");
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800784 return 0;
785 }
786 _debug_guardpage_minorder = res;
Joe Perches11705322016-03-17 14:19:50 -0700787 pr_info("Setting debug_guardpage_minorder to %lu\n", res);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800788 return 0;
789}
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700790early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800791
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700792static inline bool set_page_guard(struct zone *zone, struct page *page,
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800793 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800794{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800795 if (!debug_guardpage_enabled())
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700796 return false;
797
798 if (order >= debug_guardpage_minorder())
799 return false;
Joonsoo Kime30825f2014-12-12 16:55:49 -0800800
Vlastimil Babka3972f6b2019-07-11 20:55:13 -0700801 __SetPageGuard(page);
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800802 INIT_LIST_HEAD(&page->lru);
803 set_page_private(page, order);
804 /* Guard pages are not available for any usage */
805 __mod_zone_freepage_state(zone, -(1 << order), migratetype);
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700806
807 return true;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800808}
809
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800810static inline void clear_page_guard(struct zone *zone, struct page *page,
811 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800812{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800813 if (!debug_guardpage_enabled())
814 return;
815
Vlastimil Babka3972f6b2019-07-11 20:55:13 -0700816 __ClearPageGuard(page);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800817
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800818 set_page_private(page, 0);
819 if (!is_migrate_isolate(migratetype))
820 __mod_zone_freepage_state(zone, (1 << order), migratetype);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800821}
822#else
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700823static inline bool set_page_guard(struct zone *zone, struct page *page,
824 unsigned int order, int migratetype) { return false; }
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800825static inline void clear_page_guard(struct zone *zone, struct page *page,
826 unsigned int order, int migratetype) {}
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800827#endif
828
Vlastimil Babka04013512020-12-14 19:13:30 -0800829/*
830 * Enable static keys related to various memory debugging and hardening options.
831 * Some override others, and depend on early params that are evaluated in the
832 * order of appearance. So we need to first gather the full picture of what was
833 * enabled, and then make decisions.
834 */
835void init_mem_debugging_and_hardening(void)
836{
Sergei Trofimovich9df65f52021-04-29 23:02:11 -0700837 bool page_poisoning_requested = false;
Vlastimil Babka04013512020-12-14 19:13:30 -0800838
Vlastimil Babka8db26a32020-12-14 19:13:34 -0800839#ifdef CONFIG_PAGE_POISONING
840 /*
841 * Page poisoning is debug page alloc for some arches. If
842 * either of those options are enabled, enable poisoning.
843 */
844 if (page_poisoning_enabled() ||
845 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
Sergei Trofimovich9df65f52021-04-29 23:02:11 -0700846 debug_pagealloc_enabled())) {
Vlastimil Babka8db26a32020-12-14 19:13:34 -0800847 static_branch_enable(&_page_poisoning_enabled);
Sergei Trofimovich9df65f52021-04-29 23:02:11 -0700848 page_poisoning_requested = true;
849 }
Vlastimil Babka8db26a32020-12-14 19:13:34 -0800850#endif
851
Sergei Trofimovich69e5d322021-07-23 15:50:23 -0700852 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
853 page_poisoning_requested) {
854 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
855 "will take precedence over init_on_alloc and init_on_free\n");
856 _init_on_alloc_enabled_early = false;
857 _init_on_free_enabled_early = false;
Sergei Trofimovich9df65f52021-04-29 23:02:11 -0700858 }
Sergei Trofimovich69e5d322021-07-23 15:50:23 -0700859
860 if (_init_on_alloc_enabled_early)
861 static_branch_enable(&init_on_alloc);
862 else
863 static_branch_disable(&init_on_alloc);
864
865 if (_init_on_free_enabled_early)
866 static_branch_enable(&init_on_free);
867 else
868 static_branch_disable(&init_on_free);
Sergei Trofimovich9df65f52021-04-29 23:02:11 -0700869
Vlastimil Babka04013512020-12-14 19:13:30 -0800870#ifdef CONFIG_DEBUG_PAGEALLOC
871 if (!debug_pagealloc_enabled())
872 return;
873
874 static_branch_enable(&_debug_pagealloc_enabled);
875
876 if (!debug_guardpage_minorder())
877 return;
878
879 static_branch_enable(&_debug_guardpage_enabled);
880#endif
881}
882
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700883static inline void set_buddy_order(struct page *page, unsigned int order)
Andrew Morton6aa3001b22006-04-18 22:20:52 -0700884{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700885 set_page_private(page, order);
Nick Piggin676165a2006-04-10 11:21:48 +1000886 __SetPageBuddy(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887}
888
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890 * This function checks whether a page is free && is the buddy
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700891 * we can coalesce a page and its buddy if
Vlastimil Babka13ad59d2017-02-22 15:41:51 -0800892 * (a) the buddy is not in a hole (check before calling!) &&
Nick Piggin676165a2006-04-10 11:21:48 +1000893 * (b) the buddy is in the buddy system &&
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700894 * (c) a page and its buddy have the same order &&
895 * (d) a page and its buddy are in the same zone.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 *
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700897 * For recording whether a page is in the buddy system, we set PageBuddy.
898 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
Nick Piggin676165a2006-04-10 11:21:48 +1000899 *
900 * For recording page's order, we use page_private(page).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 */
chenqiwufe925c02020-04-01 21:09:56 -0700902static inline bool page_is_buddy(struct page *page, struct page *buddy,
Mel Gorman7aeb09f2014-06-04 16:10:21 -0700903 unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904{
chenqiwufe925c02020-04-01 21:09:56 -0700905 if (!page_is_guard(buddy) && !PageBuddy(buddy))
906 return false;
Mel Gormand34c5fa2014-06-04 16:10:10 -0700907
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700908 if (buddy_order(buddy) != order)
chenqiwufe925c02020-04-01 21:09:56 -0700909 return false;
Weijie Yang4c5018c2015-02-10 14:11:39 -0800910
chenqiwufe925c02020-04-01 21:09:56 -0700911 /*
912 * zone check is done late to avoid uselessly calculating
913 * zone/node ids for pages that could never merge.
914 */
915 if (page_zone_id(page) != page_zone_id(buddy))
916 return false;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800917
chenqiwufe925c02020-04-01 21:09:56 -0700918 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
Mel Gormand34c5fa2014-06-04 16:10:10 -0700919
chenqiwufe925c02020-04-01 21:09:56 -0700920 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921}
922
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800923#ifdef CONFIG_COMPACTION
924static inline struct capture_control *task_capc(struct zone *zone)
925{
926 struct capture_control *capc = current->capture_control;
927
Vlastimil Babkadeba0482020-08-06 23:25:16 -0700928 return unlikely(capc) &&
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800929 !(current->flags & PF_KTHREAD) &&
930 !capc->page &&
Vlastimil Babkadeba0482020-08-06 23:25:16 -0700931 capc->cc->zone == zone ? capc : NULL;
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800932}
933
934static inline bool
935compaction_capture(struct capture_control *capc, struct page *page,
936 int order, int migratetype)
937{
938 if (!capc || order != capc->cc->order)
939 return false;
940
941 /* Do not accidentally pollute CMA or isolated regions*/
942 if (is_migrate_cma(migratetype) ||
943 is_migrate_isolate(migratetype))
944 return false;
945
946 /*
Ingo Molnarf0953a12021-05-06 18:06:47 -0700947 * Do not let lower order allocations pollute a movable pageblock.
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800948 * This might let an unmovable request use a reclaimable pageblock
949 * and vice-versa but no more than normal fallback logic which can
950 * have trouble finding a high-order free page.
951 */
952 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
953 return false;
954
955 capc->page = page;
956 return true;
957}
958
959#else
960static inline struct capture_control *task_capc(struct zone *zone)
961{
962 return NULL;
963}
964
965static inline bool
966compaction_capture(struct capture_control *capc, struct page *page,
967 int order, int migratetype)
968{
969 return false;
970}
971#endif /* CONFIG_COMPACTION */
972
Alexander Duyck6ab01362020-04-06 20:04:49 -0700973/* Used for pages not on another list */
974static inline void add_to_free_list(struct page *page, struct zone *zone,
975 unsigned int order, int migratetype)
976{
977 struct free_area *area = &zone->free_area[order];
978
979 list_add(&page->lru, &area->free_list[migratetype]);
980 area->nr_free++;
981}
982
983/* Used for pages not on another list */
984static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
985 unsigned int order, int migratetype)
986{
987 struct free_area *area = &zone->free_area[order];
988
989 list_add_tail(&page->lru, &area->free_list[migratetype]);
990 area->nr_free++;
991}
992
David Hildenbrand293ffa52020-10-15 20:09:30 -0700993/*
994 * Used for pages which are on another list. Move the pages to the tail
995 * of the list - so the moved pages won't immediately be considered for
996 * allocation again (e.g., optimization for memory onlining).
997 */
Alexander Duyck6ab01362020-04-06 20:04:49 -0700998static inline void move_to_free_list(struct page *page, struct zone *zone,
999 unsigned int order, int migratetype)
1000{
1001 struct free_area *area = &zone->free_area[order];
1002
David Hildenbrand293ffa52020-10-15 20:09:30 -07001003 list_move_tail(&page->lru, &area->free_list[migratetype]);
Alexander Duyck6ab01362020-04-06 20:04:49 -07001004}
1005
1006static inline void del_page_from_free_list(struct page *page, struct zone *zone,
1007 unsigned int order)
1008{
Alexander Duyck36e66c52020-04-06 20:04:56 -07001009 /* clear reported state and update reported page count */
1010 if (page_reported(page))
1011 __ClearPageReported(page);
1012
Alexander Duyck6ab01362020-04-06 20:04:49 -07001013 list_del(&page->lru);
1014 __ClearPageBuddy(page);
1015 set_page_private(page, 0);
1016 zone->free_area[order].nr_free--;
1017}
1018
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019/*
Alexander Duycka2129f22020-04-06 20:04:45 -07001020 * If this is not the largest possible page, check if the buddy
1021 * of the next-highest order is free. If it is, it's possible
1022 * that pages are being freed that will coalesce soon. In case,
1023 * that is happening, add the free page to the tail of the list
1024 * so it's less likely to be used soon and more likely to be merged
1025 * as a higher order page
1026 */
1027static inline bool
1028buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
1029 struct page *page, unsigned int order)
1030{
1031 struct page *higher_page, *higher_buddy;
1032 unsigned long combined_pfn;
1033
1034 if (order >= MAX_ORDER - 2)
1035 return false;
1036
Alexander Duycka2129f22020-04-06 20:04:45 -07001037 combined_pfn = buddy_pfn & pfn;
1038 higher_page = page + (combined_pfn - pfn);
1039 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
1040 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
1041
Mike Rapoport859a85d2021-09-07 19:54:52 -07001042 return page_is_buddy(higher_page, higher_buddy, order + 1);
Alexander Duycka2129f22020-04-06 20:04:45 -07001043}
1044
1045/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046 * Freeing function for a buddy system allocator.
1047 *
1048 * The concept of a buddy system is to maintain direct-mapped table
1049 * (containing bit values) for memory blocks of various "orders".
1050 * The bottom level table contains the map for the smallest allocatable
1051 * units of memory (here, pages), and each level above it describes
1052 * pairs of units from the levels below, hence, "buddies".
1053 * At a high level, all that happens here is marking the table entry
1054 * at the bottom level available, and propagating the changes upward
1055 * as necessary, plus some accounting needed to play nicely with other
1056 * parts of the VM system.
1057 * At each level, we keep a list of pages, which are heads of continuous
Matthew Wilcox6e292b92018-06-07 17:08:18 -07001058 * free pages of length of (1 << order) and marked with PageBuddy.
1059 * Page's order is recorded in page_private(page) field.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 * So when we are allocating or freeing one, we can derive the state of the
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01001061 * other. That is, if we allocate a small block, and both were
1062 * free, the remainder of the region must be split into blocks.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 * If a block is freed, and its buddy is also free, then this
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01001064 * triggers coalescing into a block of larger size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +01001066 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 */
1068
Nick Piggin48db57f2006-01-08 01:00:42 -08001069static inline void __free_one_page(struct page *page,
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001070 unsigned long pfn,
Mel Gormaned0ae212009-06-16 15:32:07 -07001071 struct zone *zone, unsigned int order,
David Hildenbrandf04a5d52020-10-15 20:09:20 -07001072 int migratetype, fpi_t fpi_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073{
Mel Gorman5e1f0f02019-03-05 15:45:41 -08001074 struct capture_control *capc = task_capc(zone);
Kees Cook3f649ab2020-06-03 13:09:38 -07001075 unsigned long buddy_pfn;
Alexander Duycka2129f22020-04-06 20:04:45 -07001076 unsigned long combined_pfn;
Alexander Duycka2129f22020-04-06 20:04:45 -07001077 unsigned int max_order;
1078 struct page *buddy;
1079 bool to_tail;
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001080
Muchun Song7ad69832020-12-14 19:11:25 -08001081 max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082
Cody P Schaferd29bb972013-02-22 16:35:25 -08001083 VM_BUG_ON(!zone_is_initialized(zone));
Kirill A. Shutemov6e9f0d52015-02-11 15:25:50 -08001084 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085
Mel Gormaned0ae212009-06-16 15:32:07 -07001086 VM_BUG_ON(migratetype == -1);
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001087 if (likely(!is_migrate_isolate(migratetype)))
Joonsoo Kim8f82b552014-11-13 15:19:18 -08001088 __mod_zone_freepage_state(zone, 1 << order, migratetype);
Mel Gormaned0ae212009-06-16 15:32:07 -07001089
Vlastimil Babka76741e72017-02-22 15:41:48 -08001090 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
Sasha Levin309381fea2014-01-23 15:52:54 -08001091 VM_BUG_ON_PAGE(bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001093continue_merging:
Muchun Song7ad69832020-12-14 19:11:25 -08001094 while (order < max_order) {
Mel Gorman5e1f0f02019-03-05 15:45:41 -08001095 if (compaction_capture(capc, page, order, migratetype)) {
1096 __mod_zone_freepage_state(zone, -(1 << order),
1097 migratetype);
1098 return;
1099 }
Vlastimil Babka76741e72017-02-22 15:41:48 -08001100 buddy_pfn = __find_buddy_pfn(pfn, order);
1101 buddy = page + (buddy_pfn - pfn);
Vlastimil Babka13ad59d2017-02-22 15:41:51 -08001102
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -07001103 if (!page_is_buddy(page, buddy, order))
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001104 goto done_merging;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08001105 /*
1106 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
1107 * merge with it and move up one order.
1108 */
Dan Williamsb03641a2019-05-14 15:41:32 -07001109 if (page_is_guard(buddy))
Joonsoo Kim2847cf92014-12-12 16:55:01 -08001110 clear_page_guard(zone, buddy, order, migratetype);
Dan Williamsb03641a2019-05-14 15:41:32 -07001111 else
Alexander Duyck6ab01362020-04-06 20:04:49 -07001112 del_page_from_free_list(buddy, zone, order);
Vlastimil Babka76741e72017-02-22 15:41:48 -08001113 combined_pfn = buddy_pfn & pfn;
1114 page = page + (combined_pfn - pfn);
1115 pfn = combined_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116 order++;
1117 }
Muchun Song7ad69832020-12-14 19:11:25 -08001118 if (order < MAX_ORDER - 1) {
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001119 /* If we are here, it means order is >= pageblock_order.
1120 * We want to prevent merge between freepages on isolate
1121 * pageblock and normal pageblock. Without this, pageblock
1122 * isolation could cause incorrect freepage or CMA accounting.
1123 *
1124 * We don't want to hit this code for the more frequent
1125 * low-order merging.
1126 */
1127 if (unlikely(has_isolate_pageblock(zone))) {
1128 int buddy_mt;
1129
Vlastimil Babka76741e72017-02-22 15:41:48 -08001130 buddy_pfn = __find_buddy_pfn(pfn, order);
1131 buddy = page + (buddy_pfn - pfn);
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001132 buddy_mt = get_pageblock_migratetype(buddy);
1133
1134 if (migratetype != buddy_mt
1135 && (is_migrate_isolate(migratetype) ||
1136 is_migrate_isolate(buddy_mt)))
1137 goto done_merging;
1138 }
Muchun Song7ad69832020-12-14 19:11:25 -08001139 max_order = order + 1;
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001140 goto continue_merging;
1141 }
1142
1143done_merging:
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07001144 set_buddy_order(page, order);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -07001145
David Hildenbrand47b6a24a22020-10-15 20:09:26 -07001146 if (fpi_flags & FPI_TO_TAIL)
1147 to_tail = true;
1148 else if (is_shuffle_order(order))
Alexander Duycka2129f22020-04-06 20:04:45 -07001149 to_tail = shuffle_pick_tail();
Dan Williams97500a42019-05-14 15:41:35 -07001150 else
Alexander Duycka2129f22020-04-06 20:04:45 -07001151 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
Dan Williams97500a42019-05-14 15:41:35 -07001152
Alexander Duycka2129f22020-04-06 20:04:45 -07001153 if (to_tail)
Alexander Duyck6ab01362020-04-06 20:04:49 -07001154 add_to_free_list_tail(page, zone, order, migratetype);
Alexander Duycka2129f22020-04-06 20:04:45 -07001155 else
Alexander Duyck6ab01362020-04-06 20:04:49 -07001156 add_to_free_list(page, zone, order, migratetype);
Alexander Duyck36e66c52020-04-06 20:04:56 -07001157
1158 /* Notify page reporting subsystem of freed page */
David Hildenbrandf04a5d52020-10-15 20:09:20 -07001159 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
Alexander Duyck36e66c52020-04-06 20:04:56 -07001160 page_reporting_notify_free(order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161}
1162
Mel Gorman7bfec6f2016-05-19 17:14:15 -07001163/*
1164 * A bad page could be due to a number of fields. Instead of multiple branches,
1165 * try and check multiple fields with one check. The caller must do a detailed
1166 * check if necessary.
1167 */
1168static inline bool page_expected_state(struct page *page,
1169 unsigned long check_flags)
1170{
1171 if (unlikely(atomic_read(&page->_mapcount) != -1))
1172 return false;
1173
1174 if (unlikely((unsigned long)page->mapping |
1175 page_ref_count(page) |
1176#ifdef CONFIG_MEMCG
Muchun Song48060832021-04-29 22:56:45 -07001177 page->memcg_data |
Mel Gorman7bfec6f2016-05-19 17:14:15 -07001178#endif
1179 (page->flags & check_flags)))
1180 return false;
1181
1182 return true;
1183}
1184
Wei Yang58b7f112020-06-03 15:58:39 -07001185static const char *page_bad_reason(struct page *page, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186{
Wei Yang82a32412020-06-03 15:58:29 -07001187 const char *bad_reason = NULL;
Dave Hansenf0b791a2014-01-23 15:52:49 -08001188
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08001189 if (unlikely(atomic_read(&page->_mapcount) != -1))
Dave Hansenf0b791a2014-01-23 15:52:49 -08001190 bad_reason = "nonzero mapcount";
1191 if (unlikely(page->mapping != NULL))
1192 bad_reason = "non-NULL mapping";
Joonsoo Kimfe896d12016-03-17 14:19:26 -07001193 if (unlikely(page_ref_count(page) != 0))
Joonsoo Kim0139aa72016-05-19 17:10:49 -07001194 bad_reason = "nonzero _refcount";
Wei Yang58b7f112020-06-03 15:58:39 -07001195 if (unlikely(page->flags & flags)) {
1196 if (flags == PAGE_FLAGS_CHECK_AT_PREP)
1197 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
1198 else
1199 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
Dave Hansenf0b791a2014-01-23 15:52:49 -08001200 }
Johannes Weiner9edad6e2014-12-10 15:44:58 -08001201#ifdef CONFIG_MEMCG
Muchun Song48060832021-04-29 22:56:45 -07001202 if (unlikely(page->memcg_data))
Johannes Weiner9edad6e2014-12-10 15:44:58 -08001203 bad_reason = "page still charged to cgroup";
1204#endif
Wei Yang58b7f112020-06-03 15:58:39 -07001205 return bad_reason;
Mel Gormanbb552ac2016-05-19 17:14:18 -07001206}
1207
Wei Yang58b7f112020-06-03 15:58:39 -07001208static void check_free_page_bad(struct page *page)
1209{
1210 bad_page(page,
1211 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
Mel Gormanbb552ac2016-05-19 17:14:18 -07001212}
1213
Wei Yang534fe5e2020-06-03 15:58:36 -07001214static inline int check_free_page(struct page *page)
Mel Gormanbb552ac2016-05-19 17:14:18 -07001215{
Mel Gormanda838d42016-05-19 17:14:21 -07001216 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
Mel Gormanbb552ac2016-05-19 17:14:18 -07001217 return 0;
Mel Gormanbb552ac2016-05-19 17:14:18 -07001218
1219 /* Something has gone sideways, find it */
Wei Yang0d0c48a2020-06-03 15:58:33 -07001220 check_free_page_bad(page);
Mel Gorman7bfec6f2016-05-19 17:14:15 -07001221 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222}
1223
Mel Gorman4db75482016-05-19 17:14:32 -07001224static int free_tail_pages_check(struct page *head_page, struct page *page)
1225{
1226 int ret = 1;
1227
1228 /*
1229 * We rely page->lru.next never has bit 0 set, unless the page
1230 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
1231 */
1232 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1233
1234 if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1235 ret = 0;
1236 goto out;
1237 }
1238 switch (page - head_page) {
1239 case 1:
Matthew Wilcox4da19842018-06-07 17:08:50 -07001240 /* the first tail page: ->mapping may be compound_mapcount() */
Mel Gorman4db75482016-05-19 17:14:32 -07001241 if (unlikely(compound_mapcount(page))) {
Wei Yang82a32412020-06-03 15:58:29 -07001242 bad_page(page, "nonzero compound_mapcount");
Mel Gorman4db75482016-05-19 17:14:32 -07001243 goto out;
1244 }
1245 break;
1246 case 2:
1247 /*
1248 * the second tail page: ->mapping is
Matthew Wilcoxfa3015b2018-06-07 17:08:42 -07001249 * deferred_list.next -- ignore value.
Mel Gorman4db75482016-05-19 17:14:32 -07001250 */
1251 break;
1252 default:
1253 if (page->mapping != TAIL_MAPPING) {
Wei Yang82a32412020-06-03 15:58:29 -07001254 bad_page(page, "corrupted mapping in tail page");
Mel Gorman4db75482016-05-19 17:14:32 -07001255 goto out;
1256 }
1257 break;
1258 }
1259 if (unlikely(!PageTail(page))) {
Wei Yang82a32412020-06-03 15:58:29 -07001260 bad_page(page, "PageTail not set");
Mel Gorman4db75482016-05-19 17:14:32 -07001261 goto out;
1262 }
1263 if (unlikely(compound_head(page) != head_page)) {
Wei Yang82a32412020-06-03 15:58:29 -07001264 bad_page(page, "compound_head not consistent");
Mel Gorman4db75482016-05-19 17:14:32 -07001265 goto out;
1266 }
1267 ret = 0;
1268out:
1269 page->mapping = NULL;
1270 clear_compound_head(page);
1271 return ret;
1272}
1273
Peter Collingbourne013bb592021-06-02 16:52:29 -07001274static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags)
Alexander Potapenko64713842019-07-11 20:59:19 -07001275{
1276 int i;
1277
Peter Collingbourne013bb592021-06-02 16:52:29 -07001278 if (zero_tags) {
1279 for (i = 0; i < numpages; i++)
1280 tag_clear_highpage(page + i);
1281 return;
1282 }
1283
Qian Cai9e15afa2020-08-06 23:25:54 -07001284 /* s390's use of memset() could override KASAN redzones. */
1285 kasan_disable_current();
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -08001286 for (i = 0; i < numpages; i++) {
Andrey Konovalovacb35b12021-01-23 21:01:43 -08001287 u8 tag = page_kasan_tag(page + i);
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -08001288 page_kasan_tag_reset(page + i);
Alexander Potapenko64713842019-07-11 20:59:19 -07001289 clear_highpage(page + i);
Andrey Konovalovacb35b12021-01-23 21:01:43 -08001290 page_kasan_tag_set(page + i, tag);
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -08001291 }
Qian Cai9e15afa2020-08-06 23:25:54 -07001292 kasan_enable_current();
Alexander Potapenko64713842019-07-11 20:59:19 -07001293}
1294
Mel Gormane2769db2016-05-19 17:14:38 -07001295static __always_inline bool free_pages_prepare(struct page *page,
Andrey Konovalov2c335682021-04-29 22:59:52 -07001296 unsigned int order, bool check_free, fpi_t fpi_flags)
Mel Gormane2769db2016-05-19 17:14:38 -07001297{
1298 int bad = 0;
Peter Collingbournec275c5c2021-06-02 16:52:30 -07001299 bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags);
Mel Gormane2769db2016-05-19 17:14:38 -07001300
1301 VM_BUG_ON_PAGE(PageTail(page), page);
1302
1303 trace_mm_page_free(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001304
Oscar Salvador79f5f8f2020-10-15 20:07:09 -07001305 if (unlikely(PageHWPoison(page)) && !order) {
1306 /*
1307 * Do not let hwpoison pages hit pcplists/buddy
1308 * Untie memcg state and reset page's owner
1309 */
Roman Gushchin18b2db32020-12-01 13:58:30 -08001310 if (memcg_kmem_enabled() && PageMemcgKmem(page))
Oscar Salvador79f5f8f2020-10-15 20:07:09 -07001311 __memcg_kmem_uncharge_page(page, order);
1312 reset_page_owner(page, order);
Pasha Tatashindf4e8172022-01-14 14:06:37 -08001313 page_table_check_free(page, order);
Oscar Salvador79f5f8f2020-10-15 20:07:09 -07001314 return false;
1315 }
1316
Mel Gormane2769db2016-05-19 17:14:38 -07001317 /*
1318 * Check tail pages before head page information is cleared to
1319 * avoid checking PageCompound for order-0 pages.
1320 */
1321 if (unlikely(order)) {
1322 bool compound = PageCompound(page);
1323 int i;
1324
1325 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1326
Yang Shieac96c32021-10-28 14:36:11 -07001327 if (compound) {
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001328 ClearPageDoubleMap(page);
Yang Shieac96c32021-10-28 14:36:11 -07001329 ClearPageHasHWPoisoned(page);
1330 }
Mel Gormane2769db2016-05-19 17:14:38 -07001331 for (i = 1; i < (1 << order); i++) {
1332 if (compound)
1333 bad += free_tail_pages_check(page, page + i);
Wei Yang534fe5e2020-06-03 15:58:36 -07001334 if (unlikely(check_free_page(page + i))) {
Mel Gormane2769db2016-05-19 17:14:38 -07001335 bad++;
1336 continue;
1337 }
1338 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1339 }
1340 }
Minchan Kimbda807d2016-07-26 15:23:05 -07001341 if (PageMappingFlags(page))
Mel Gormane2769db2016-05-19 17:14:38 -07001342 page->mapping = NULL;
Roman Gushchin18b2db32020-12-01 13:58:30 -08001343 if (memcg_kmem_enabled() && PageMemcgKmem(page))
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07001344 __memcg_kmem_uncharge_page(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001345 if (check_free)
Wei Yang534fe5e2020-06-03 15:58:36 -07001346 bad += check_free_page(page);
Mel Gormane2769db2016-05-19 17:14:38 -07001347 if (bad)
1348 return false;
1349
1350 page_cpupid_reset_last(page);
1351 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1352 reset_page_owner(page, order);
Pasha Tatashindf4e8172022-01-14 14:06:37 -08001353 page_table_check_free(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001354
1355 if (!PageHighMem(page)) {
1356 debug_check_no_locks_freed(page_address(page),
1357 PAGE_SIZE << order);
1358 debug_check_no_obj_freed(page_address(page),
1359 PAGE_SIZE << order);
1360 }
Alexander Potapenko64713842019-07-11 20:59:19 -07001361
Vlastimil Babka8db26a32020-12-14 19:13:34 -08001362 kernel_poison_pages(page, 1 << order);
1363
Qian Cai234fdce2019-10-06 17:58:25 -07001364 /*
Andrey Konovalov1bb5eab2021-04-29 23:00:02 -07001365 * As memory initialization might be integrated into KASAN,
1366 * kasan_free_pages and kernel_init_free_pages must be
1367 * kept together to avoid discrepancies in behavior.
1368 *
Andrey Konovalovf9d79e82021-03-12 21:08:10 -08001369 * With hardware tag-based KASAN, memory tags must be set before the
1370 * page becomes unavailable via debug_pagealloc or arch_free_page.
1371 */
Peter Collingbourne7a3b8352021-06-02 16:52:28 -07001372 if (kasan_has_integrated_init()) {
1373 if (!skip_kasan_poison)
1374 kasan_free_pages(page, order);
1375 } else {
1376 bool init = want_init_on_free();
1377
1378 if (init)
Peter Collingbourne013bb592021-06-02 16:52:29 -07001379 kernel_init_free_pages(page, 1 << order, false);
Peter Collingbourne7a3b8352021-06-02 16:52:28 -07001380 if (!skip_kasan_poison)
1381 kasan_poison_pages(page, order, init);
1382 }
Andrey Konovalovf9d79e82021-03-12 21:08:10 -08001383
1384 /*
Qian Cai234fdce2019-10-06 17:58:25 -07001385 * arch_free_page() can make the page's contents inaccessible. s390
1386 * does this. So nothing which can access the page's contents should
1387 * happen after this.
1388 */
1389 arch_free_page(page, order);
1390
Mike Rapoport77bc7fd2020-12-14 19:10:20 -08001391 debug_pagealloc_unmap_pages(page, 1 << order);
Rick Edgecombed6332692019-04-25 17:11:35 -07001392
Mel Gormane2769db2016-05-19 17:14:38 -07001393 return true;
1394}
Mel Gorman4db75482016-05-19 17:14:32 -07001395
1396#ifdef CONFIG_DEBUG_VM
Vlastimil Babka4462b322019-07-11 20:55:09 -07001397/*
1398 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
1399 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when
1400 * moved from pcp lists to free lists.
1401 */
Mel Gorman44042b42021-06-28 19:43:08 -07001402static bool free_pcp_prepare(struct page *page, unsigned int order)
Mel Gorman4db75482016-05-19 17:14:32 -07001403{
Mel Gorman44042b42021-06-28 19:43:08 -07001404 return free_pages_prepare(page, order, true, FPI_NONE);
Mel Gorman4db75482016-05-19 17:14:32 -07001405}
1406
Vlastimil Babka4462b322019-07-11 20:55:09 -07001407static bool bulkfree_pcp_prepare(struct page *page)
Mel Gorman4db75482016-05-19 17:14:32 -07001408{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001409 if (debug_pagealloc_enabled_static())
Wei Yang534fe5e2020-06-03 15:58:36 -07001410 return check_free_page(page);
Vlastimil Babka4462b322019-07-11 20:55:09 -07001411 else
1412 return false;
Mel Gorman4db75482016-05-19 17:14:32 -07001413}
1414#else
Vlastimil Babka4462b322019-07-11 20:55:09 -07001415/*
1416 * With DEBUG_VM disabled, order-0 pages being freed are checked only when
1417 * moving from pcp lists to free list in order to reduce overhead. With
1418 * debug_pagealloc enabled, they are checked also immediately when being freed
1419 * to the pcp lists.
1420 */
Mel Gorman44042b42021-06-28 19:43:08 -07001421static bool free_pcp_prepare(struct page *page, unsigned int order)
Mel Gorman4db75482016-05-19 17:14:32 -07001422{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001423 if (debug_pagealloc_enabled_static())
Mel Gorman44042b42021-06-28 19:43:08 -07001424 return free_pages_prepare(page, order, true, FPI_NONE);
Vlastimil Babka4462b322019-07-11 20:55:09 -07001425 else
Mel Gorman44042b42021-06-28 19:43:08 -07001426 return free_pages_prepare(page, order, false, FPI_NONE);
Mel Gorman4db75482016-05-19 17:14:32 -07001427}
1428
1429static bool bulkfree_pcp_prepare(struct page *page)
1430{
Wei Yang534fe5e2020-06-03 15:58:36 -07001431 return check_free_page(page);
Mel Gorman4db75482016-05-19 17:14:32 -07001432}
1433#endif /* CONFIG_DEBUG_VM */
1434
Aaron Lu97334162018-04-05 16:24:14 -07001435static inline void prefetch_buddy(struct page *page)
1436{
1437 unsigned long pfn = page_to_pfn(page);
1438 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1439 struct page *buddy = page + (buddy_pfn - pfn);
1440
1441 prefetch(buddy);
1442}
1443
Linus Torvalds1da177e2005-04-16 15:20:36 -07001444/*
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001445 * Frees a number of pages from the PCP lists
Miaohe Lin7cba6302021-11-05 13:40:08 -07001446 * Assumes all pages on list are in same zone.
Renaud Lienhart207f36e2005-09-10 00:26:59 -07001447 * count is the number of pages to free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 */
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001449static void free_pcppages_bulk(struct zone *zone, int count,
1450 struct per_cpu_pages *pcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451{
Mel Gorman44042b42021-06-28 19:43:08 -07001452 int pindex = 0;
Mel Gormana6f9edd62009-09-21 17:03:20 -07001453 int batch_free = 0;
Mel Gorman44042b42021-06-28 19:43:08 -07001454 int nr_freed = 0;
1455 unsigned int order;
Vlastimil Babka5c3ad2e2020-12-14 19:10:50 -08001456 int prefetch_nr = READ_ONCE(pcp->batch);
Mel Gorman37779992016-05-19 17:13:58 -07001457 bool isolated_pageblocks;
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001458 struct page *page, *tmp;
1459 LIST_HEAD(head);
Mel Gormanf2260e62009-06-16 15:32:13 -07001460
Charan Teja Reddy88e8ac12020-08-20 17:42:27 -07001461 /*
1462 * Ensure proper count is passed which otherwise would stuck in the
1463 * below while (list_empty(list)) loop.
1464 */
1465 count = min(pcp->count, count);
Mel Gorman44042b42021-06-28 19:43:08 -07001466 while (count > 0) {
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001467 struct list_head *list;
Nick Piggin48db57f2006-01-08 01:00:42 -08001468
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001469 /*
Mel Gormana6f9edd62009-09-21 17:03:20 -07001470 * Remove pages from lists in a round-robin fashion. A
1471 * batch_free count is maintained that is incremented when an
1472 * empty list is encountered. This is so more pages are freed
1473 * off fuller lists instead of spinning excessively around empty
1474 * lists
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001475 */
1476 do {
Mel Gormana6f9edd62009-09-21 17:03:20 -07001477 batch_free++;
Mel Gorman44042b42021-06-28 19:43:08 -07001478 if (++pindex == NR_PCP_LISTS)
1479 pindex = 0;
1480 list = &pcp->lists[pindex];
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001481 } while (list_empty(list));
1482
Namhyung Kim1d168712011-03-22 16:32:45 -07001483 /* This is the only non-empty list. Free them all. */
Mel Gorman44042b42021-06-28 19:43:08 -07001484 if (batch_free == NR_PCP_LISTS)
Mel Gormane5b31ac2016-05-19 17:14:24 -07001485 batch_free = count;
Namhyung Kim1d168712011-03-22 16:32:45 -07001486
Mel Gorman44042b42021-06-28 19:43:08 -07001487 order = pindex_to_order(pindex);
1488 BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH));
Mel Gormana6f9edd62009-09-21 17:03:20 -07001489 do {
Geliang Tanga16601c2016-01-14 15:20:30 -08001490 page = list_last_entry(list, struct page, lru);
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001491 /* must delete to avoid corrupting pcp list */
Mel Gormana6f9edd62009-09-21 17:03:20 -07001492 list_del(&page->lru);
Mel Gorman44042b42021-06-28 19:43:08 -07001493 nr_freed += 1 << order;
1494 count -= 1 << order;
Vlastimil Babkaaa016d12015-09-08 15:01:22 -07001495
Mel Gorman4db75482016-05-19 17:14:32 -07001496 if (bulkfree_pcp_prepare(page))
1497 continue;
1498
Mel Gorman44042b42021-06-28 19:43:08 -07001499 /* Encode order with the migratetype */
1500 page->index <<= NR_PCP_ORDER_WIDTH;
1501 page->index |= order;
1502
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001503 list_add_tail(&page->lru, &head);
Aaron Lu97334162018-04-05 16:24:14 -07001504
1505 /*
1506 * We are going to put the page back to the global
1507 * pool, prefetch its buddy to speed up later access
1508 * under zone->lock. It is believed the overhead of
1509 * an additional test and calculating buddy_pfn here
1510 * can be offset by reduced memory latency later. To
1511 * avoid excessive prefetching due to large count, only
1512 * prefetch buddy for the first pcp->batch nr of pages.
1513 */
Vlastimil Babka5c3ad2e2020-12-14 19:10:50 -08001514 if (prefetch_nr) {
Aaron Lu97334162018-04-05 16:24:14 -07001515 prefetch_buddy(page);
Vlastimil Babka5c3ad2e2020-12-14 19:10:50 -08001516 prefetch_nr--;
1517 }
Mel Gorman44042b42021-06-28 19:43:08 -07001518 } while (count > 0 && --batch_free && !list_empty(list));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519 }
Mel Gorman44042b42021-06-28 19:43:08 -07001520 pcp->count -= nr_freed;
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001521
Mel Gormandbbee9d2021-06-28 19:41:41 -07001522 /*
1523 * local_lock_irq held so equivalent to spin_lock_irqsave for
1524 * both PREEMPT_RT and non-PREEMPT_RT configurations.
1525 */
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001526 spin_lock(&zone->lock);
1527 isolated_pageblocks = has_isolate_pageblock(zone);
1528
1529 /*
1530 * Use safe version since after __free_one_page(),
1531 * page->lru.next will not point to original list.
1532 */
1533 list_for_each_entry_safe(page, tmp, &head, lru) {
1534 int mt = get_pcppage_migratetype(page);
Mel Gorman44042b42021-06-28 19:43:08 -07001535
1536 /* mt has been encoded with the order (see above) */
1537 order = mt & NR_PCP_ORDER_MASK;
1538 mt >>= NR_PCP_ORDER_WIDTH;
1539
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001540 /* MIGRATE_ISOLATE page should not go to pcplists */
1541 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1542 /* Pageblock could have been isolated meanwhile */
1543 if (unlikely(isolated_pageblocks))
1544 mt = get_pageblock_migratetype(page);
1545
Mel Gorman44042b42021-06-28 19:43:08 -07001546 __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE);
1547 trace_mm_page_pcpu_drain(page, order, mt);
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001548 }
Mel Gormand34b0732017-04-20 14:37:43 -07001549 spin_unlock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550}
1551
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001552static void free_one_page(struct zone *zone,
1553 struct page *page, unsigned long pfn,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07001554 unsigned int order,
David Hildenbrand7fef4312020-10-15 20:09:35 -07001555 int migratetype, fpi_t fpi_flags)
Nick Piggin48db57f2006-01-08 01:00:42 -08001556{
Mel Gormandf1acc82021-06-28 19:42:00 -07001557 unsigned long flags;
1558
1559 spin_lock_irqsave(&zone->lock, flags);
Joonsoo Kimad53f922014-11-13 15:19:11 -08001560 if (unlikely(has_isolate_pageblock(zone) ||
1561 is_migrate_isolate(migratetype))) {
1562 migratetype = get_pfnblock_migratetype(page, pfn);
Joonsoo Kimad53f922014-11-13 15:19:11 -08001563 }
David Hildenbrand7fef4312020-10-15 20:09:35 -07001564 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
Mel Gormandf1acc82021-06-28 19:42:00 -07001565 spin_unlock_irqrestore(&zone->lock, flags);
Nick Piggin48db57f2006-01-08 01:00:42 -08001566}
1567
Robin Holt1e8ce832015-06-30 14:56:45 -07001568static void __meminit __init_single_page(struct page *page, unsigned long pfn,
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001569 unsigned long zone, int nid)
Robin Holt1e8ce832015-06-30 14:56:45 -07001570{
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001571 mm_zero_struct_page(page);
Robin Holt1e8ce832015-06-30 14:56:45 -07001572 set_page_links(page, zone, nid, pfn);
Robin Holt1e8ce832015-06-30 14:56:45 -07001573 init_page_count(page);
1574 page_mapcount_reset(page);
1575 page_cpupid_reset_last(page);
Andrey Konovalov2813b9c2018-12-28 00:30:57 -08001576 page_kasan_tag_reset(page);
Robin Holt1e8ce832015-06-30 14:56:45 -07001577
Robin Holt1e8ce832015-06-30 14:56:45 -07001578 INIT_LIST_HEAD(&page->lru);
1579#ifdef WANT_PAGE_VIRTUAL
1580 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1581 if (!is_highmem_idx(zone))
1582 set_page_address(page, __va(pfn << PAGE_SHIFT));
1583#endif
1584}
1585
Mel Gorman7e18adb2015-06-30 14:57:05 -07001586#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Arnd Bergmann57148a62017-10-03 16:15:10 -07001587static void __meminit init_reserved_page(unsigned long pfn)
Mel Gorman7e18adb2015-06-30 14:57:05 -07001588{
1589 pg_data_t *pgdat;
1590 int nid, zid;
1591
1592 if (!early_page_uninitialised(pfn))
1593 return;
1594
1595 nid = early_pfn_to_nid(pfn);
1596 pgdat = NODE_DATA(nid);
1597
1598 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1599 struct zone *zone = &pgdat->node_zones[zid];
1600
Miaohe Lin86fb05b2021-11-05 13:40:11 -07001601 if (zone_spans_pfn(zone, pfn))
Mel Gorman7e18adb2015-06-30 14:57:05 -07001602 break;
1603 }
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001604 __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001605}
1606#else
1607static inline void init_reserved_page(unsigned long pfn)
1608{
1609}
1610#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1611
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001612/*
1613 * Initialised pages do not have PageReserved set. This function is
1614 * called for each range allocated by the bootmem allocator and
1615 * marks the pages PageReserved. The remaining valid pages are later
1616 * sent to the buddy page allocator.
1617 */
Stefan Bader4b50bcc2016-05-20 16:58:38 -07001618void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001619{
1620 unsigned long start_pfn = PFN_DOWN(start);
1621 unsigned long end_pfn = PFN_UP(end);
1622
Mel Gorman7e18adb2015-06-30 14:57:05 -07001623 for (; start_pfn < end_pfn; start_pfn++) {
1624 if (pfn_valid(start_pfn)) {
1625 struct page *page = pfn_to_page(start_pfn);
1626
1627 init_reserved_page(start_pfn);
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -08001628
1629 /* Avoid false-positive PageTail() */
1630 INIT_LIST_HEAD(&page->lru);
1631
Alexander Duyckd483da52018-10-26 15:07:48 -07001632 /*
1633 * no need for atomic set_bit because the struct
1634 * page is not visible yet so nobody should
1635 * access it yet.
1636 */
1637 __SetPageReserved(page);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001638 }
1639 }
Nathan Zimmer92923ca2015-06-30 14:56:48 -07001640}
1641
David Hildenbrand7fef4312020-10-15 20:09:35 -07001642static void __free_pages_ok(struct page *page, unsigned int order,
1643 fpi_t fpi_flags)
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001644{
Mel Gormand34b0732017-04-20 14:37:43 -07001645 unsigned long flags;
Minchan Kim95e34412012-10-08 16:32:11 -07001646 int migratetype;
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001647 unsigned long pfn = page_to_pfn(page);
Mel Gorman56f0e662021-06-28 19:41:57 -07001648 struct zone *zone = page_zone(page);
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001649
Andrey Konovalov2c335682021-04-29 22:59:52 -07001650 if (!free_pages_prepare(page, order, true, fpi_flags))
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001651 return;
1652
Mel Gormancfc47a22014-06-04 16:10:19 -07001653 migratetype = get_pfnblock_migratetype(page, pfn);
Mel Gormandbbee9d2021-06-28 19:41:41 -07001654
Mel Gorman56f0e662021-06-28 19:41:57 -07001655 spin_lock_irqsave(&zone->lock, flags);
Mel Gorman56f0e662021-06-28 19:41:57 -07001656 if (unlikely(has_isolate_pageblock(zone) ||
1657 is_migrate_isolate(migratetype))) {
1658 migratetype = get_pfnblock_migratetype(page, pfn);
1659 }
1660 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
1661 spin_unlock_irqrestore(&zone->lock, flags);
Mel Gorman90249992021-06-28 19:42:03 -07001662
Mel Gormand34b0732017-04-20 14:37:43 -07001663 __count_vm_events(PGFREE, 1 << order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664}
1665
Arun KSa9cd4102019-03-05 15:42:14 -08001666void __free_pages_core(struct page *page, unsigned int order)
David Howellsa226f6c2006-01-06 00:11:08 -08001667{
Johannes Weinerc3993072012-01-10 15:08:10 -08001668 unsigned int nr_pages = 1 << order;
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001669 struct page *p = page;
Johannes Weinerc3993072012-01-10 15:08:10 -08001670 unsigned int loop;
David Howellsa226f6c2006-01-06 00:11:08 -08001671
David Hildenbrand7fef4312020-10-15 20:09:35 -07001672 /*
1673 * When initializing the memmap, __init_single_page() sets the refcount
1674 * of all pages to 1 ("allocated"/"not free"). We have to set the
1675 * refcount of all involved pages to 0.
1676 */
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001677 prefetchw(p);
1678 for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1679 prefetchw(p + 1);
Johannes Weinerc3993072012-01-10 15:08:10 -08001680 __ClearPageReserved(p);
1681 set_page_count(p, 0);
David Howellsa226f6c2006-01-06 00:11:08 -08001682 }
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001683 __ClearPageReserved(p);
1684 set_page_count(p, 0);
Johannes Weinerc3993072012-01-10 15:08:10 -08001685
Arun KS9705bea2018-12-28 00:34:24 -08001686 atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
David Hildenbrand7fef4312020-10-15 20:09:35 -07001687
1688 /*
1689 * Bypass PCP and place fresh pages right to the tail, primarily
1690 * relevant for memory onlining.
1691 */
Andrey Konovalov2c335682021-04-29 22:59:52 -07001692 __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON);
David Howellsa226f6c2006-01-06 00:11:08 -08001693}
1694
Mike Rapoporta9ee6cf2021-06-28 19:43:01 -07001695#ifdef CONFIG_NUMA
Mel Gorman7ace9912015-08-06 15:46:13 -07001696
Mike Rapoport03e92a52020-12-14 19:09:32 -08001697/*
1698 * During memory init memblocks map pfns to nids. The search is expensive and
1699 * this caches recent lookups. The implementation of __early_pfn_to_nid
1700 * treats start/end as pfns.
1701 */
1702struct mminit_pfnnid_cache {
1703 unsigned long last_start;
1704 unsigned long last_end;
1705 int last_nid;
1706};
Mel Gorman75a592a2015-06-30 14:56:59 -07001707
Mike Rapoport03e92a52020-12-14 19:09:32 -08001708static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
Mike Rapoport6f24fbd2020-06-03 15:56:57 -07001709
1710/*
1711 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
1712 */
Mike Rapoport03e92a52020-12-14 19:09:32 -08001713static int __meminit __early_pfn_to_nid(unsigned long pfn,
Mike Rapoport6f24fbd2020-06-03 15:56:57 -07001714 struct mminit_pfnnid_cache *state)
1715{
1716 unsigned long start_pfn, end_pfn;
1717 int nid;
1718
1719 if (state->last_start <= pfn && pfn < state->last_end)
1720 return state->last_nid;
1721
1722 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
1723 if (nid != NUMA_NO_NODE) {
1724 state->last_start = start_pfn;
1725 state->last_end = end_pfn;
1726 state->last_nid = nid;
1727 }
1728
1729 return nid;
1730}
Mike Rapoport6f24fbd2020-06-03 15:56:57 -07001731
Mel Gorman75a592a2015-06-30 14:56:59 -07001732int __meminit early_pfn_to_nid(unsigned long pfn)
1733{
Mel Gorman7ace9912015-08-06 15:46:13 -07001734 static DEFINE_SPINLOCK(early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001735 int nid;
1736
Mel Gorman7ace9912015-08-06 15:46:13 -07001737 spin_lock(&early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001738 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
Mel Gorman7ace9912015-08-06 15:46:13 -07001739 if (nid < 0)
Mel Gormane4568d32016-07-14 12:07:20 -07001740 nid = first_online_node;
Mel Gorman7ace9912015-08-06 15:46:13 -07001741 spin_unlock(&early_pfn_lock);
1742
1743 return nid;
Mel Gorman75a592a2015-06-30 14:56:59 -07001744}
Mike Rapoporta9ee6cf2021-06-28 19:43:01 -07001745#endif /* CONFIG_NUMA */
Mel Gorman75a592a2015-06-30 14:56:59 -07001746
Mike Rapoport7c2ee342018-10-30 15:09:36 -07001747void __init memblock_free_pages(struct page *page, unsigned long pfn,
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001748 unsigned int order)
1749{
1750 if (early_page_uninitialised(pfn))
1751 return;
Arun KSa9cd4102019-03-05 15:42:14 -08001752 __free_pages_core(page, order);
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001753}
1754
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001755/*
1756 * Check that the whole (or subset of) a pageblock given by the interval of
1757 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
Mike Rapoport859a85d2021-09-07 19:54:52 -07001758 * with the migration of free compaction scanner.
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001759 *
1760 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1761 *
1762 * It's possible on some configurations to have a setup like node0 node1 node0
1763 * i.e. it's possible that all pages within a zones range of pages do not
1764 * belong to a single zone. We assume that a border between node0 and node1
1765 * can occur within a single pageblock, but not a node0 node1 node0
1766 * interleaving within a single pageblock. It is therefore sufficient to check
1767 * the first and last page of a pageblock and avoid checking each individual
1768 * page in a pageblock.
1769 */
1770struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1771 unsigned long end_pfn, struct zone *zone)
1772{
1773 struct page *start_page;
1774 struct page *end_page;
1775
1776 /* end_pfn is one past the range we are checking */
1777 end_pfn--;
1778
1779 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1780 return NULL;
1781
Michal Hocko2d070ea2017-07-06 15:37:56 -07001782 start_page = pfn_to_online_page(start_pfn);
1783 if (!start_page)
1784 return NULL;
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001785
1786 if (page_zone(start_page) != zone)
1787 return NULL;
1788
1789 end_page = pfn_to_page(end_pfn);
1790
1791 /* This gives a shorter code than deriving page_zone(end_page) */
1792 if (page_zone_id(start_page) != page_zone_id(end_page))
1793 return NULL;
1794
1795 return start_page;
1796}
1797
1798void set_zone_contiguous(struct zone *zone)
1799{
1800 unsigned long block_start_pfn = zone->zone_start_pfn;
1801 unsigned long block_end_pfn;
1802
1803 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1804 for (; block_start_pfn < zone_end_pfn(zone);
1805 block_start_pfn = block_end_pfn,
1806 block_end_pfn += pageblock_nr_pages) {
1807
1808 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1809
1810 if (!__pageblock_pfn_to_page(block_start_pfn,
1811 block_end_pfn, zone))
1812 return;
David Hildenbrande84fe992020-05-07 18:35:46 -07001813 cond_resched();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001814 }
1815
1816 /* We confirm that there is no hole */
1817 zone->contiguous = true;
1818}
1819
1820void clear_zone_contiguous(struct zone *zone)
1821{
1822 zone->contiguous = false;
1823}
1824
Mel Gorman7e18adb2015-06-30 14:57:05 -07001825#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001826static void __init deferred_free_range(unsigned long pfn,
1827 unsigned long nr_pages)
Mel Gormana4de83d2015-06-30 14:57:16 -07001828{
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001829 struct page *page;
1830 unsigned long i;
Mel Gormana4de83d2015-06-30 14:57:16 -07001831
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001832 if (!nr_pages)
Mel Gormana4de83d2015-06-30 14:57:16 -07001833 return;
1834
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001835 page = pfn_to_page(pfn);
1836
Mel Gormana4de83d2015-06-30 14:57:16 -07001837 /* Free a large naturally-aligned chunk if possible */
Xishi Qiue7801492016-10-07 16:58:09 -07001838 if (nr_pages == pageblock_nr_pages &&
1839 (pfn & (pageblock_nr_pages - 1)) == 0) {
Mel Gormanac5d2532015-06-30 14:57:20 -07001840 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Arun KSa9cd4102019-03-05 15:42:14 -08001841 __free_pages_core(page, pageblock_order);
Mel Gormana4de83d2015-06-30 14:57:16 -07001842 return;
1843 }
1844
Xishi Qiue7801492016-10-07 16:58:09 -07001845 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1846 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1847 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Arun KSa9cd4102019-03-05 15:42:14 -08001848 __free_pages_core(page, 0);
Xishi Qiue7801492016-10-07 16:58:09 -07001849 }
Mel Gormana4de83d2015-06-30 14:57:16 -07001850}
1851
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001852/* Completion tracking for deferred_init_memmap() threads */
1853static atomic_t pgdat_init_n_undone __initdata;
1854static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1855
1856static inline void __init pgdat_init_report_one_done(void)
1857{
1858 if (atomic_dec_and_test(&pgdat_init_n_undone))
1859 complete(&pgdat_init_all_done_comp);
1860}
Mel Gorman0e1cc952015-06-30 14:57:27 -07001861
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001862/*
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001863 * Returns true if page needs to be initialized or freed to buddy allocator.
1864 *
1865 * First we check if pfn is valid on architectures where it is possible to have
1866 * holes within pageblock_nr_pages. On systems where it is not possible, this
1867 * function is optimized out.
1868 *
1869 * Then, we check if a current large page is valid by only checking the validity
1870 * of the head pfn.
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001871 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001872static inline bool __init deferred_pfn_valid(unsigned long pfn)
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001873{
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001874 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1875 return false;
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001876 return true;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001877}
1878
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001879/*
1880 * Free pages to buddy allocator. Try to free aligned pages in
1881 * pageblock_nr_pages sizes.
1882 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001883static void __init deferred_free_pages(unsigned long pfn,
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001884 unsigned long end_pfn)
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001885{
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001886 unsigned long nr_pgmask = pageblock_nr_pages - 1;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001887 unsigned long nr_free = 0;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001888
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001889 for (; pfn < end_pfn; pfn++) {
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001890 if (!deferred_pfn_valid(pfn)) {
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001891 deferred_free_range(pfn - nr_free, nr_free);
1892 nr_free = 0;
1893 } else if (!(pfn & nr_pgmask)) {
1894 deferred_free_range(pfn - nr_free, nr_free);
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001895 nr_free = 1;
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001896 } else {
1897 nr_free++;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001898 }
1899 }
1900 /* Free the last block of pages to allocator */
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001901 deferred_free_range(pfn - nr_free, nr_free);
1902}
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001903
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001904/*
1905 * Initialize struct pages. We minimize pfn page lookups and scheduler checks
1906 * by performing it only once every pageblock_nr_pages.
1907 * Return number of pages initialized.
1908 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001909static unsigned long __init deferred_init_pages(struct zone *zone,
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001910 unsigned long pfn,
1911 unsigned long end_pfn)
1912{
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001913 unsigned long nr_pgmask = pageblock_nr_pages - 1;
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001914 int nid = zone_to_nid(zone);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001915 unsigned long nr_pages = 0;
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001916 int zid = zone_idx(zone);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001917 struct page *page = NULL;
1918
1919 for (; pfn < end_pfn; pfn++) {
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001920 if (!deferred_pfn_valid(pfn)) {
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001921 page = NULL;
1922 continue;
1923 } else if (!page || !(pfn & nr_pgmask)) {
1924 page = pfn_to_page(pfn);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001925 } else {
1926 page++;
1927 }
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001928 __init_single_page(page, pfn, zid, nid);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001929 nr_pages++;
1930 }
1931 return (nr_pages);
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001932}
1933
Alexander Duyck0e56aca2019-05-13 17:21:20 -07001934/*
1935 * This function is meant to pre-load the iterator for the zone init.
1936 * Specifically it walks through the ranges until we are caught up to the
1937 * first_init_pfn value and exits there. If we never encounter the value we
1938 * return false indicating there are no valid ranges left.
1939 */
1940static bool __init
1941deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
1942 unsigned long *spfn, unsigned long *epfn,
1943 unsigned long first_init_pfn)
1944{
1945 u64 j;
1946
1947 /*
1948 * Start out by walking through the ranges in this zone that have
1949 * already been initialized. We don't need to do anything with them
1950 * so we just need to flush them out of the system.
1951 */
1952 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
1953 if (*epfn <= first_init_pfn)
1954 continue;
1955 if (*spfn < first_init_pfn)
1956 *spfn = first_init_pfn;
1957 *i = j;
1958 return true;
1959 }
1960
1961 return false;
1962}
1963
1964/*
1965 * Initialize and free pages. We do it in two loops: first we initialize
1966 * struct page, then free to buddy allocator, because while we are
1967 * freeing pages we can access pages that are ahead (computing buddy
1968 * page in __free_one_page()).
1969 *
1970 * In order to try and keep some memory in the cache we have the loop
1971 * broken along max page order boundaries. This way we will not cause
1972 * any issues with the buddy page computation.
1973 */
1974static unsigned long __init
1975deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
1976 unsigned long *end_pfn)
1977{
1978 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
1979 unsigned long spfn = *start_pfn, epfn = *end_pfn;
1980 unsigned long nr_pages = 0;
1981 u64 j = *i;
1982
1983 /* First we loop through and initialize the page values */
1984 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
1985 unsigned long t;
1986
1987 if (mo_pfn <= *start_pfn)
1988 break;
1989
1990 t = min(mo_pfn, *end_pfn);
1991 nr_pages += deferred_init_pages(zone, *start_pfn, t);
1992
1993 if (mo_pfn < *end_pfn) {
1994 *start_pfn = mo_pfn;
1995 break;
1996 }
1997 }
1998
1999 /* Reset values and now loop through freeing pages as needed */
2000 swap(j, *i);
2001
2002 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
2003 unsigned long t;
2004
2005 if (mo_pfn <= spfn)
2006 break;
2007
2008 t = min(mo_pfn, epfn);
2009 deferred_free_pages(spfn, t);
2010
2011 if (mo_pfn <= epfn)
2012 break;
2013 }
2014
2015 return nr_pages;
2016}
2017
Daniel Jordane4443142020-06-03 15:59:51 -07002018static void __init
2019deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2020 void *arg)
2021{
2022 unsigned long spfn, epfn;
2023 struct zone *zone = arg;
2024 u64 i;
2025
2026 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
2027
2028 /*
2029 * Initialize and free pages in MAX_ORDER sized increments so that we
2030 * can avoid introducing any issues with the buddy allocator.
2031 */
2032 while (spfn < end_pfn) {
2033 deferred_init_maxorder(&i, zone, &spfn, &epfn);
2034 cond_resched();
2035 }
2036}
2037
Daniel Jordanecd09652020-06-03 15:59:55 -07002038/* An arch may override for more concurrency. */
2039__weak int __init
2040deferred_page_init_max_threads(const struct cpumask *node_cpumask)
2041{
2042 return 1;
2043}
2044
Mel Gorman7e18adb2015-06-30 14:57:05 -07002045/* Initialise remaining memory on a node */
Mel Gorman0e1cc952015-06-30 14:57:27 -07002046static int __init deferred_init_memmap(void *data)
Mel Gorman7e18adb2015-06-30 14:57:05 -07002047{
Mel Gorman0e1cc952015-06-30 14:57:27 -07002048 pg_data_t *pgdat = data;
Mel Gorman0e1cc952015-06-30 14:57:27 -07002049 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
Daniel Jordan89c7c402020-06-03 15:59:47 -07002050 unsigned long spfn = 0, epfn = 0;
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002051 unsigned long first_init_pfn, flags;
2052 unsigned long start = jiffies;
2053 struct zone *zone;
Daniel Jordane4443142020-06-03 15:59:51 -07002054 int zid, max_threads;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08002055 u64 i;
Mel Gorman7e18adb2015-06-30 14:57:05 -07002056
Mel Gorman0e1cc952015-06-30 14:57:27 -07002057 /* Bind memory initialisation thread to a local node if possible */
2058 if (!cpumask_empty(cpumask))
2059 set_cpus_allowed_ptr(current, cpumask);
Mel Gorman7e18adb2015-06-30 14:57:05 -07002060
Pavel Tatashin3a2d7fa2018-04-05 16:22:27 -07002061 pgdat_resize_lock(pgdat, &flags);
2062 first_init_pfn = pgdat->first_deferred_pfn;
2063 if (first_init_pfn == ULONG_MAX) {
2064 pgdat_resize_unlock(pgdat, &flags);
2065 pgdat_init_report_one_done();
2066 return 0;
2067 }
2068
Mel Gorman7e18adb2015-06-30 14:57:05 -07002069 /* Sanity check boundaries */
2070 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
2071 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
2072 pgdat->first_deferred_pfn = ULONG_MAX;
2073
Pavel Tatashin3d060852020-06-03 15:59:24 -07002074 /*
2075 * Once we unlock here, the zone cannot be grown anymore, thus if an
2076 * interrupt thread must allocate this early in boot, zone must be
2077 * pre-grown prior to start of deferred page initialization.
2078 */
2079 pgdat_resize_unlock(pgdat, &flags);
2080
Mel Gorman7e18adb2015-06-30 14:57:05 -07002081 /* Only the highest zone is deferred so find it */
2082 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2083 zone = pgdat->node_zones + zid;
2084 if (first_init_pfn < zone_end_pfn(zone))
2085 break;
2086 }
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002087
2088 /* If the zone is empty somebody else may have cleared out the zone */
2089 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2090 first_init_pfn))
2091 goto zone_empty;
Mel Gorman7e18adb2015-06-30 14:57:05 -07002092
Daniel Jordanecd09652020-06-03 15:59:55 -07002093 max_threads = deferred_page_init_max_threads(cpumask);
Mel Gorman7e18adb2015-06-30 14:57:05 -07002094
Daniel Jordan117003c2020-06-03 15:59:20 -07002095 while (spfn < epfn) {
Daniel Jordane4443142020-06-03 15:59:51 -07002096 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
2097 struct padata_mt_job job = {
2098 .thread_fn = deferred_init_memmap_chunk,
2099 .fn_arg = zone,
2100 .start = spfn,
2101 .size = epfn_align - spfn,
2102 .align = PAGES_PER_SECTION,
2103 .min_chunk = PAGES_PER_SECTION,
2104 .max_threads = max_threads,
2105 };
2106
2107 padata_do_multithreaded(&job);
2108 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2109 epfn_align);
Daniel Jordan117003c2020-06-03 15:59:20 -07002110 }
Mel Gorman7e18adb2015-06-30 14:57:05 -07002111zone_empty:
Mel Gorman7e18adb2015-06-30 14:57:05 -07002112 /* Sanity check that the next zone really is unpopulated */
2113 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
2114
Daniel Jordan89c7c402020-06-03 15:59:47 -07002115 pr_info("node %d deferred pages initialised in %ums\n",
2116 pgdat->node_id, jiffies_to_msecs(jiffies - start));
Nicolai Stanged3cd1312015-08-06 15:46:16 -07002117
2118 pgdat_init_report_one_done();
Mel Gorman0e1cc952015-06-30 14:57:27 -07002119 return 0;
2120}
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002121
2122/*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002123 * If this zone has deferred pages, try to grow it by initializing enough
2124 * deferred pages to satisfy the allocation specified by order, rounded up to
2125 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
2126 * of SECTION_SIZE bytes by initializing struct pages in increments of
2127 * PAGES_PER_SECTION * sizeof(struct page) bytes.
2128 *
2129 * Return true when zone was grown, otherwise return false. We return true even
2130 * when we grow less than requested, to let the caller decide if there are
2131 * enough pages to satisfy the allocation.
2132 *
2133 * Note: We use noinline because this function is needed only during boot, and
2134 * it is called from a __ref function _deferred_grow_zone. This way we are
2135 * making sure that it is not inlined into permanent text section.
2136 */
2137static noinline bool __init
2138deferred_grow_zone(struct zone *zone, unsigned int order)
2139{
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002140 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
Alexander Duyck837566e2019-05-13 17:21:17 -07002141 pg_data_t *pgdat = zone->zone_pgdat;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002142 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002143 unsigned long spfn, epfn, flags;
2144 unsigned long nr_pages = 0;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002145 u64 i;
2146
2147 /* Only the last zone may have deferred pages */
2148 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2149 return false;
2150
2151 pgdat_resize_lock(pgdat, &flags);
2152
2153 /*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002154 * If someone grew this zone while we were waiting for spinlock, return
2155 * true, as there might be enough pages already.
2156 */
2157 if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2158 pgdat_resize_unlock(pgdat, &flags);
2159 return true;
2160 }
2161
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002162 /* If the zone is empty somebody else may have cleared out the zone */
2163 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2164 first_deferred_pfn)) {
2165 pgdat->first_deferred_pfn = ULONG_MAX;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002166 pgdat_resize_unlock(pgdat, &flags);
Juergen Grossb9705d82019-07-04 15:14:36 -07002167 /* Retry only once. */
2168 return first_deferred_pfn != ULONG_MAX;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002169 }
2170
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002171 /*
2172 * Initialize and free pages in MAX_ORDER sized increments so
2173 * that we can avoid introducing any issues with the buddy
2174 * allocator.
2175 */
2176 while (spfn < epfn) {
2177 /* update our first deferred PFN for this section */
2178 first_deferred_pfn = spfn;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002179
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002180 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
Daniel Jordan117003c2020-06-03 15:59:20 -07002181 touch_nmi_watchdog();
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002182
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002183 /* We should only stop along section boundaries */
2184 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2185 continue;
2186
2187 /* If our quota has been met we can stop here */
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002188 if (nr_pages >= nr_pages_needed)
2189 break;
2190 }
2191
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002192 pgdat->first_deferred_pfn = spfn;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002193 pgdat_resize_unlock(pgdat, &flags);
2194
2195 return nr_pages > 0;
2196}
2197
2198/*
2199 * deferred_grow_zone() is __init, but it is called from
2200 * get_page_from_freelist() during early boot until deferred_pages permanently
2201 * disables this call. This is why we have refdata wrapper to avoid warning,
2202 * and to ensure that the function body gets unloaded.
2203 */
2204static bool __ref
2205_deferred_grow_zone(struct zone *zone, unsigned int order)
2206{
2207 return deferred_grow_zone(zone, order);
2208}
2209
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002210#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
Mel Gorman0e1cc952015-06-30 14:57:27 -07002211
2212void __init page_alloc_init_late(void)
2213{
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002214 struct zone *zone;
Dan Williamse900a912019-05-14 15:41:28 -07002215 int nid;
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002216
2217#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Mel Gorman0e1cc952015-06-30 14:57:27 -07002218
Nicolai Stanged3cd1312015-08-06 15:46:16 -07002219 /* There will be num_node_state(N_MEMORY) threads */
2220 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
Mel Gorman0e1cc952015-06-30 14:57:27 -07002221 for_each_node_state(nid, N_MEMORY) {
Mel Gorman0e1cc952015-06-30 14:57:27 -07002222 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2223 }
2224
2225 /* Block until all are initialised */
Nicolai Stanged3cd1312015-08-06 15:46:16 -07002226 wait_for_completion(&pgdat_init_all_done_comp);
Mel Gorman4248b0d2015-08-06 15:46:20 -07002227
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002228 /*
2229 * We initialized the rest of the deferred pages. Permanently disable
2230 * on-demand struct page initialization.
2231 */
2232 static_branch_disable(&deferred_pages);
2233
Mel Gorman4248b0d2015-08-06 15:46:20 -07002234 /* Reinit limits that are based on free pages after the kernel is up */
2235 files_maxfiles_init();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002236#endif
Mike Rapoport350e88b2019-05-13 17:22:59 -07002237
Lin Fengba8f3582020-12-14 19:11:19 -08002238 buffer_init();
2239
Pavel Tatashin3010f872017-08-18 15:16:05 -07002240 /* Discard memblock private memory */
2241 memblock_discard();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002242
Dan Williamse900a912019-05-14 15:41:28 -07002243 for_each_node_state(nid, N_MEMORY)
2244 shuffle_free_memory(NODE_DATA(nid));
2245
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002246 for_each_populated_zone(zone)
2247 set_zone_contiguous(zone);
Mel Gorman7e18adb2015-06-30 14:57:05 -07002248}
Mel Gorman7e18adb2015-06-30 14:57:05 -07002249
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002250#ifdef CONFIG_CMA
Li Zhong9cf510a2013-08-23 13:52:52 +08002251/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002252void __init init_cma_reserved_pageblock(struct page *page)
2253{
2254 unsigned i = pageblock_nr_pages;
2255 struct page *p = page;
2256
2257 do {
2258 __ClearPageReserved(p);
2259 set_page_count(p, 0);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09002260 } while (++p, --i);
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002261
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002262 set_pageblock_migratetype(page, MIGRATE_CMA);
Michal Nazarewiczdc783272014-07-02 15:22:35 -07002263
2264 if (pageblock_order >= MAX_ORDER) {
2265 i = pageblock_nr_pages;
2266 p = page;
2267 do {
2268 set_page_refcounted(p);
2269 __free_pages(p, MAX_ORDER - 1);
2270 p += MAX_ORDER_NR_PAGES;
2271 } while (i -= MAX_ORDER_NR_PAGES);
2272 } else {
2273 set_page_refcounted(page);
2274 __free_pages(page, pageblock_order);
2275 }
2276
Jiang Liu3dcc0572013-07-03 15:03:21 -07002277 adjust_managed_page_count(page, pageblock_nr_pages);
David Hildenbrand3c381db2021-02-25 17:16:40 -08002278 page_zone(page)->cma_pages += pageblock_nr_pages;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002279}
2280#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281
2282/*
2283 * The order of subdivision here is critical for the IO subsystem.
2284 * Please do not alter this order without good reasons and regression
2285 * testing. Specifically, as large blocks of memory are subdivided,
2286 * the order in which smaller blocks are delivered depends on the order
2287 * they're subdivided in this function. This is the primary factor
2288 * influencing the order in which pages are delivered to the IO
2289 * subsystem according to empirical testing, and this is also justified
2290 * by considering the behavior of a buddy system containing a single
2291 * large block of memory acted on by a series of small allocations.
2292 * This behavior is a critical factor in sglist merging's success.
2293 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +01002294 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -07002295 */
Nick Piggin085cc7d52006-01-06 00:11:01 -08002296static inline void expand(struct zone *zone, struct page *page,
Alexander Duyck6ab01362020-04-06 20:04:49 -07002297 int low, int high, int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002298{
2299 unsigned long size = 1 << high;
2300
2301 while (high > low) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302 high--;
2303 size >>= 1;
Sasha Levin309381fea2014-01-23 15:52:54 -08002304 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08002305
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07002306 /*
2307 * Mark as guard pages (or page), that will allow to
2308 * merge back to allocator when buddy will be freed.
2309 * Corresponding page table entries will not be touched,
2310 * pages will stay not present in virtual address space
2311 */
2312 if (set_page_guard(zone, &page[size], high, migratetype))
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08002313 continue;
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07002314
Alexander Duyck6ab01362020-04-06 20:04:49 -07002315 add_to_free_list(&page[size], zone, high, migratetype);
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002316 set_buddy_order(&page[size], high);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002318}
2319
Vlastimil Babka4e611802016-05-19 17:14:41 -07002320static void check_new_page_bad(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321{
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07002322 if (unlikely(page->flags & __PG_HWPOISON)) {
Naoya Horiguchie570f562016-05-20 16:58:50 -07002323 /* Don't complain about hwpoisoned pages */
2324 page_mapcount_reset(page); /* remove PageBuddy */
2325 return;
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07002326 }
Wei Yang58b7f112020-06-03 15:58:39 -07002327
2328 bad_page(page,
2329 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
Vlastimil Babka4e611802016-05-19 17:14:41 -07002330}
2331
2332/*
2333 * This page is about to be returned from the page allocator
2334 */
2335static inline int check_new_page(struct page *page)
2336{
2337 if (likely(page_expected_state(page,
2338 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
2339 return 0;
2340
2341 check_new_page_bad(page);
2342 return 1;
Wu Fengguang2a7684a2009-09-16 11:50:12 +02002343}
2344
Mel Gorman479f8542016-05-19 17:14:35 -07002345#ifdef CONFIG_DEBUG_VM
Vlastimil Babka4462b322019-07-11 20:55:09 -07002346/*
2347 * With DEBUG_VM enabled, order-0 pages are checked for expected state when
2348 * being allocated from pcp lists. With debug_pagealloc also enabled, they are
2349 * also checked when pcp lists are refilled from the free lists.
2350 */
2351static inline bool check_pcp_refill(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002352{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002353 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07002354 return check_new_page(page);
2355 else
2356 return false;
Mel Gorman479f8542016-05-19 17:14:35 -07002357}
2358
Vlastimil Babka4462b322019-07-11 20:55:09 -07002359static inline bool check_new_pcp(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002360{
2361 return check_new_page(page);
2362}
2363#else
Vlastimil Babka4462b322019-07-11 20:55:09 -07002364/*
2365 * With DEBUG_VM disabled, free order-0 pages are checked for expected state
2366 * when pcp lists are being refilled from the free lists. With debug_pagealloc
2367 * enabled, they are also checked when being allocated from the pcp lists.
2368 */
2369static inline bool check_pcp_refill(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002370{
2371 return check_new_page(page);
2372}
Vlastimil Babka4462b322019-07-11 20:55:09 -07002373static inline bool check_new_pcp(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002374{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002375 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07002376 return check_new_page(page);
2377 else
2378 return false;
Mel Gorman479f8542016-05-19 17:14:35 -07002379}
2380#endif /* CONFIG_DEBUG_VM */
2381
2382static bool check_new_pages(struct page *page, unsigned int order)
2383{
2384 int i;
2385 for (i = 0; i < (1 << order); i++) {
2386 struct page *p = page + i;
2387
2388 if (unlikely(check_new_page(p)))
2389 return true;
2390 }
2391
2392 return false;
2393}
2394
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002395inline void post_alloc_hook(struct page *page, unsigned int order,
2396 gfp_t gfp_flags)
2397{
2398 set_page_private(page, 0);
2399 set_page_refcounted(page);
2400
2401 arch_alloc_page(page, order);
Mike Rapoport77bc7fd2020-12-14 19:10:20 -08002402 debug_pagealloc_map_pages(page, 1 << order);
David Hildenbrand862b6de2020-12-14 19:11:15 -08002403
Andrey Konovalov1bb5eab2021-04-29 23:00:02 -07002404 /*
2405 * Page unpoisoning must happen before memory initialization.
2406 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO
2407 * allocations and the page unpoisoning code will complain.
2408 */
2409 kernel_unpoison_pages(page, 1 << order);
2410
2411 /*
2412 * As memory initialization might be integrated into KASAN,
2413 * kasan_alloc_pages and kernel_init_free_pages must be
2414 * kept together to avoid discrepancies in behavior.
2415 */
Peter Collingbourne7a3b8352021-06-02 16:52:28 -07002416 if (kasan_has_integrated_init()) {
2417 kasan_alloc_pages(page, order, gfp_flags);
2418 } else {
2419 bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
2420
2421 kasan_unpoison_pages(page, order, init);
2422 if (init)
Peter Collingbourne013bb592021-06-02 16:52:29 -07002423 kernel_init_free_pages(page, 1 << order,
2424 gfp_flags & __GFP_ZEROTAGS);
Peter Collingbourne7a3b8352021-06-02 16:52:28 -07002425 }
Andrey Konovalov1bb5eab2021-04-29 23:00:02 -07002426
2427 set_page_owner(page, order, gfp_flags);
Pasha Tatashindf4e8172022-01-14 14:06:37 -08002428 page_table_check_alloc(page, order);
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002429}
2430
Mel Gorman479f8542016-05-19 17:14:35 -07002431static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Mel Gormanc6038442016-05-19 17:13:38 -07002432 unsigned int alloc_flags)
Wu Fengguang2a7684a2009-09-16 11:50:12 +02002433{
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002434 post_alloc_hook(page, order, gfp_flags);
Nick Piggin17cf4402006-03-22 00:08:41 -08002435
Nick Piggin17cf4402006-03-22 00:08:41 -08002436 if (order && (gfp_flags & __GFP_COMP))
2437 prep_compound_page(page, order);
2438
Vlastimil Babka75379192015-02-11 15:25:38 -08002439 /*
Michal Hocko2f064f32015-08-21 14:11:51 -07002440 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
Vlastimil Babka75379192015-02-11 15:25:38 -08002441 * allocate the page. The expectation is that the caller is taking
2442 * steps that will free more memory. The caller should avoid the page
2443 * being used for !PFMEMALLOC purposes.
2444 */
Michal Hocko2f064f32015-08-21 14:11:51 -07002445 if (alloc_flags & ALLOC_NO_WATERMARKS)
2446 set_page_pfmemalloc(page);
2447 else
2448 clear_page_pfmemalloc(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449}
2450
Mel Gorman56fd56b2007-10-16 01:25:58 -07002451/*
2452 * Go through the free lists for the given migratetype and remove
2453 * the smallest available page from the freelists
2454 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002455static __always_inline
Mel Gorman728ec982009-06-16 15:32:04 -07002456struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
Mel Gorman56fd56b2007-10-16 01:25:58 -07002457 int migratetype)
2458{
2459 unsigned int current_order;
Pintu Kumarb8af2942013-09-11 14:20:34 -07002460 struct free_area *area;
Mel Gorman56fd56b2007-10-16 01:25:58 -07002461 struct page *page;
2462
2463 /* Find a page of the appropriate size in the preferred list */
2464 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2465 area = &(zone->free_area[current_order]);
Dan Williamsb03641a2019-05-14 15:41:32 -07002466 page = get_page_from_free_area(area, migratetype);
Geliang Tanga16601c2016-01-14 15:20:30 -08002467 if (!page)
2468 continue;
Alexander Duyck6ab01362020-04-06 20:04:49 -07002469 del_page_from_free_list(page, zone, current_order);
2470 expand(zone, page, order, current_order, migratetype);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002471 set_pcppage_migratetype(page, migratetype);
Mel Gorman56fd56b2007-10-16 01:25:58 -07002472 return page;
2473 }
2474
2475 return NULL;
2476}
2477
2478
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002479/*
2480 * This array describes the order lists are fallen back to when
2481 * the free lists for the desirable migrate type are depleted
2482 */
Wei Yangda415662020-08-06 23:25:58 -07002483static int fallbacks[MIGRATE_TYPES][3] = {
Mel Gorman974a7862015-11-06 16:28:34 -08002484 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
Mel Gorman974a7862015-11-06 16:28:34 -08002485 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
Huang Shijie7ead3342018-12-28 00:34:46 -08002486 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
Joonsoo Kimdc676472015-04-14 15:45:15 -07002487#ifdef CONFIG_CMA
Mel Gorman974a7862015-11-06 16:28:34 -08002488 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002489#endif
Minchan Kim194159f2013-02-22 16:33:58 -08002490#ifdef CONFIG_MEMORY_ISOLATION
Mel Gorman974a7862015-11-06 16:28:34 -08002491 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */
Minchan Kim194159f2013-02-22 16:33:58 -08002492#endif
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002493};
2494
Joonsoo Kimdc676472015-04-14 15:45:15 -07002495#ifdef CONFIG_CMA
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002496static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
Joonsoo Kimdc676472015-04-14 15:45:15 -07002497 unsigned int order)
2498{
2499 return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2500}
2501#else
2502static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2503 unsigned int order) { return NULL; }
2504#endif
2505
Mel Gormanc361be52007-10-16 01:25:51 -07002506/*
David Hildenbrand293ffa52020-10-15 20:09:30 -07002507 * Move the free pages in a range to the freelist tail of the requested type.
Mel Gormand9c23402007-10-16 01:26:01 -07002508 * Note that start_page and end_pages are not aligned on a pageblock
Mel Gormanc361be52007-10-16 01:25:51 -07002509 * boundary. If alignment is required, use move_freepages_block()
2510 */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002511static int move_freepages(struct zone *zone,
Kefeng Wang39ddb992021-04-29 23:01:36 -07002512 unsigned long start_pfn, unsigned long end_pfn,
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002513 int migratetype, int *num_movable)
Mel Gormanc361be52007-10-16 01:25:51 -07002514{
2515 struct page *page;
Kefeng Wang39ddb992021-04-29 23:01:36 -07002516 unsigned long pfn;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08002517 unsigned int order;
Mel Gormand1003132007-10-16 01:26:00 -07002518 int pages_moved = 0;
Mel Gormanc361be52007-10-16 01:25:51 -07002519
Kefeng Wang39ddb992021-04-29 23:01:36 -07002520 for (pfn = start_pfn; pfn <= end_pfn;) {
Kefeng Wang39ddb992021-04-29 23:01:36 -07002521 page = pfn_to_page(pfn);
Mel Gormanc361be52007-10-16 01:25:51 -07002522 if (!PageBuddy(page)) {
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002523 /*
2524 * We assume that pages that could be isolated for
2525 * migration are movable. But we don't actually try
2526 * isolating, as that would be expensive.
2527 */
2528 if (num_movable &&
2529 (PageLRU(page) || __PageMovable(page)))
2530 (*num_movable)++;
Kefeng Wang39ddb992021-04-29 23:01:36 -07002531 pfn++;
Mel Gormanc361be52007-10-16 01:25:51 -07002532 continue;
2533 }
2534
David Rientjescd961032019-08-24 17:54:40 -07002535 /* Make sure we are not inadvertently changing nodes */
2536 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2537 VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2538
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002539 order = buddy_order(page);
Alexander Duyck6ab01362020-04-06 20:04:49 -07002540 move_to_free_list(page, zone, order, migratetype);
Kefeng Wang39ddb992021-04-29 23:01:36 -07002541 pfn += 1 << order;
Mel Gormand1003132007-10-16 01:26:00 -07002542 pages_moved += 1 << order;
Mel Gormanc361be52007-10-16 01:25:51 -07002543 }
2544
Mel Gormand1003132007-10-16 01:26:00 -07002545 return pages_moved;
Mel Gormanc361be52007-10-16 01:25:51 -07002546}
2547
Minchan Kimee6f5092012-07-31 16:43:50 -07002548int move_freepages_block(struct zone *zone, struct page *page,
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002549 int migratetype, int *num_movable)
Mel Gormanc361be52007-10-16 01:25:51 -07002550{
Kefeng Wang39ddb992021-04-29 23:01:36 -07002551 unsigned long start_pfn, end_pfn, pfn;
Mel Gormanc361be52007-10-16 01:25:51 -07002552
David Rientjes4a222122018-10-26 15:09:24 -07002553 if (num_movable)
2554 *num_movable = 0;
2555
Kefeng Wang39ddb992021-04-29 23:01:36 -07002556 pfn = page_to_pfn(page);
2557 start_pfn = pfn & ~(pageblock_nr_pages - 1);
Mel Gormand9c23402007-10-16 01:26:01 -07002558 end_pfn = start_pfn + pageblock_nr_pages - 1;
Mel Gormanc361be52007-10-16 01:25:51 -07002559
2560 /* Do not cross zone boundaries */
Cody P Schafer108bcc92013-02-22 16:35:23 -08002561 if (!zone_spans_pfn(zone, start_pfn))
Kefeng Wang39ddb992021-04-29 23:01:36 -07002562 start_pfn = pfn;
Cody P Schafer108bcc92013-02-22 16:35:23 -08002563 if (!zone_spans_pfn(zone, end_pfn))
Mel Gormanc361be52007-10-16 01:25:51 -07002564 return 0;
2565
Kefeng Wang39ddb992021-04-29 23:01:36 -07002566 return move_freepages(zone, start_pfn, end_pfn, migratetype,
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002567 num_movable);
Mel Gormanc361be52007-10-16 01:25:51 -07002568}
2569
Mel Gorman2f66a682009-09-21 17:02:31 -07002570static void change_pageblock_range(struct page *pageblock_page,
2571 int start_order, int migratetype)
2572{
2573 int nr_pageblocks = 1 << (start_order - pageblock_order);
2574
2575 while (nr_pageblocks--) {
2576 set_pageblock_migratetype(pageblock_page, migratetype);
2577 pageblock_page += pageblock_nr_pages;
2578 }
2579}
2580
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002581/*
Vlastimil Babka9c0415e2015-02-11 15:28:21 -08002582 * When we are falling back to another migratetype during allocation, try to
2583 * steal extra free pages from the same pageblocks to satisfy further
2584 * allocations, instead of polluting multiple pageblocks.
2585 *
2586 * If we are stealing a relatively large buddy page, it is likely there will
2587 * be more free pages in the pageblock, so try to steal them all. For
2588 * reclaimable and unmovable allocations, we steal regardless of page size,
2589 * as fragmentation caused by those allocations polluting movable pageblocks
2590 * is worse than movable allocations stealing from unmovable and reclaimable
2591 * pageblocks.
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002592 */
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002593static bool can_steal_fallback(unsigned int order, int start_mt)
2594{
2595 /*
2596 * Leaving this order check is intended, although there is
2597 * relaxed order check in next check. The reason is that
2598 * we can actually steal whole pageblock if this condition met,
2599 * but, below check doesn't guarantee it and that is just heuristic
2600 * so could be changed anytime.
2601 */
2602 if (order >= pageblock_order)
2603 return true;
2604
2605 if (order >= pageblock_order / 2 ||
2606 start_mt == MIGRATE_RECLAIMABLE ||
2607 start_mt == MIGRATE_UNMOVABLE ||
2608 page_group_by_mobility_disabled)
2609 return true;
2610
2611 return false;
2612}
2613
Johannes Weiner597c8922020-12-14 19:12:15 -08002614static inline bool boost_watermark(struct zone *zone)
Mel Gorman1c308442018-12-28 00:35:52 -08002615{
2616 unsigned long max_boost;
2617
2618 if (!watermark_boost_factor)
Johannes Weiner597c8922020-12-14 19:12:15 -08002619 return false;
Henry Willard14f69142020-05-07 18:36:27 -07002620 /*
2621 * Don't bother in zones that are unlikely to produce results.
2622 * On small machines, including kdump capture kernels running
2623 * in a small area, boosting the watermark can cause an out of
2624 * memory situation immediately.
2625 */
2626 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
Johannes Weiner597c8922020-12-14 19:12:15 -08002627 return false;
Mel Gorman1c308442018-12-28 00:35:52 -08002628
2629 max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2630 watermark_boost_factor, 10000);
Mel Gorman94b33342019-02-20 22:19:49 -08002631
2632 /*
2633 * high watermark may be uninitialised if fragmentation occurs
2634 * very early in boot so do not boost. We do not fall
2635 * through and boost by pageblock_nr_pages as failing
2636 * allocations that early means that reclaim is not going
2637 * to help and it may even be impossible to reclaim the
2638 * boosted watermark resulting in a hang.
2639 */
2640 if (!max_boost)
Johannes Weiner597c8922020-12-14 19:12:15 -08002641 return false;
Mel Gorman94b33342019-02-20 22:19:49 -08002642
Mel Gorman1c308442018-12-28 00:35:52 -08002643 max_boost = max(pageblock_nr_pages, max_boost);
2644
2645 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2646 max_boost);
Johannes Weiner597c8922020-12-14 19:12:15 -08002647
2648 return true;
Mel Gorman1c308442018-12-28 00:35:52 -08002649}
2650
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002651/*
2652 * This function implements actual steal behaviour. If order is large enough,
2653 * we can steal whole pageblock. If not, we first move freepages in this
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002654 * pageblock to our migratetype and determine how many already-allocated pages
2655 * are there in the pageblock with a compatible migratetype. If at least half
2656 * of pages are free or compatible, we can change migratetype of the pageblock
2657 * itself, so pages freed in the future will be put on the correct free list.
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002658 */
2659static void steal_suitable_fallback(struct zone *zone, struct page *page,
Mel Gorman1c308442018-12-28 00:35:52 -08002660 unsigned int alloc_flags, int start_type, bool whole_block)
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002661{
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002662 unsigned int current_order = buddy_order(page);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002663 int free_pages, movable_pages, alike_pages;
2664 int old_block_type;
2665
2666 old_block_type = get_pageblock_migratetype(page);
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002667
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002668 /*
2669 * This can happen due to races and we want to prevent broken
2670 * highatomic accounting.
2671 */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002672 if (is_migrate_highatomic(old_block_type))
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002673 goto single_page;
2674
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002675 /* Take ownership for orders >= pageblock_order */
2676 if (current_order >= pageblock_order) {
2677 change_pageblock_range(page, current_order, start_type);
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002678 goto single_page;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002679 }
2680
Mel Gorman1c308442018-12-28 00:35:52 -08002681 /*
2682 * Boost watermarks to increase reclaim pressure to reduce the
2683 * likelihood of future fallbacks. Wake kswapd now as the node
2684 * may be balanced overall and kswapd will not wake naturally.
2685 */
Johannes Weiner597c8922020-12-14 19:12:15 -08002686 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
Mel Gorman73444bc2019-01-08 15:23:39 -08002687 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
Mel Gorman1c308442018-12-28 00:35:52 -08002688
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002689 /* We are not allowed to try stealing from the whole block */
2690 if (!whole_block)
2691 goto single_page;
2692
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002693 free_pages = move_freepages_block(zone, page, start_type,
2694 &movable_pages);
2695 /*
2696 * Determine how many pages are compatible with our allocation.
2697 * For movable allocation, it's the number of movable pages which
2698 * we just obtained. For other types it's a bit more tricky.
2699 */
2700 if (start_type == MIGRATE_MOVABLE) {
2701 alike_pages = movable_pages;
2702 } else {
2703 /*
2704 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2705 * to MOVABLE pageblock, consider all non-movable pages as
2706 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2707 * vice versa, be conservative since we can't distinguish the
2708 * exact migratetype of non-movable pages.
2709 */
2710 if (old_block_type == MIGRATE_MOVABLE)
2711 alike_pages = pageblock_nr_pages
2712 - (free_pages + movable_pages);
2713 else
2714 alike_pages = 0;
2715 }
2716
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002717 /* moving whole block can fail due to zone boundary conditions */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002718 if (!free_pages)
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002719 goto single_page;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002720
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002721 /*
2722 * If a sufficient number of pages in the block are either free or of
2723 * comparable migratability as our allocation, claim the whole block.
2724 */
2725 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002726 page_group_by_mobility_disabled)
2727 set_pageblock_migratetype(page, start_type);
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002728
2729 return;
2730
2731single_page:
Alexander Duyck6ab01362020-04-06 20:04:49 -07002732 move_to_free_list(page, zone, current_order, start_type);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002733}
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002734
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002735/*
2736 * Check whether there is a suitable fallback freepage with requested order.
2737 * If only_stealable is true, this function returns fallback_mt only if
2738 * we can steal other freepages all together. This would help to reduce
2739 * fragmentation due to mixed migratetype pages in one pageblock.
2740 */
2741int find_suitable_fallback(struct free_area *area, unsigned int order,
2742 int migratetype, bool only_stealable, bool *can_steal)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002743{
2744 int i;
2745 int fallback_mt;
2746
2747 if (area->nr_free == 0)
2748 return -1;
2749
2750 *can_steal = false;
2751 for (i = 0;; i++) {
2752 fallback_mt = fallbacks[migratetype][i];
Mel Gorman974a7862015-11-06 16:28:34 -08002753 if (fallback_mt == MIGRATE_TYPES)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002754 break;
2755
Dan Williamsb03641a2019-05-14 15:41:32 -07002756 if (free_area_empty(area, fallback_mt))
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002757 continue;
2758
2759 if (can_steal_fallback(order, migratetype))
2760 *can_steal = true;
2761
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002762 if (!only_stealable)
2763 return fallback_mt;
2764
2765 if (*can_steal)
2766 return fallback_mt;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002767 }
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002768
2769 return -1;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002770}
2771
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002772/*
2773 * Reserve a pageblock for exclusive use of high-order atomic allocations if
2774 * there are no empty page blocks that contain a page with a suitable order
2775 */
2776static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2777 unsigned int alloc_order)
2778{
2779 int mt;
2780 unsigned long max_managed, flags;
2781
2782 /*
2783 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2784 * Check is race-prone but harmless.
2785 */
Arun KS9705bea2018-12-28 00:34:24 -08002786 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002787 if (zone->nr_reserved_highatomic >= max_managed)
2788 return;
2789
2790 spin_lock_irqsave(&zone->lock, flags);
2791
2792 /* Recheck the nr_reserved_highatomic limit under the lock */
2793 if (zone->nr_reserved_highatomic >= max_managed)
2794 goto out_unlock;
2795
2796 /* Yoink! */
2797 mt = get_pageblock_migratetype(page);
Xishi Qiua6ffdc02017-05-03 14:52:52 -07002798 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2799 && !is_migrate_cma(mt)) {
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002800 zone->nr_reserved_highatomic += pageblock_nr_pages;
2801 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002802 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002803 }
2804
2805out_unlock:
2806 spin_unlock_irqrestore(&zone->lock, flags);
2807}
2808
2809/*
2810 * Used when an allocation is about to fail under memory pressure. This
2811 * potentially hurts the reliability of high-order allocations when under
2812 * intense memory pressure but failed atomic allocations should be easier
2813 * to recover from than an OOM.
Minchan Kim29fac032016-12-12 16:42:14 -08002814 *
2815 * If @force is true, try to unreserve a pageblock even though highatomic
2816 * pageblock is exhausted.
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002817 */
Minchan Kim29fac032016-12-12 16:42:14 -08002818static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2819 bool force)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002820{
2821 struct zonelist *zonelist = ac->zonelist;
2822 unsigned long flags;
2823 struct zoneref *z;
2824 struct zone *zone;
2825 struct page *page;
2826 int order;
Minchan Kim04c87162016-12-12 16:42:11 -08002827 bool ret;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002828
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002829 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002830 ac->nodemask) {
Minchan Kim29fac032016-12-12 16:42:14 -08002831 /*
2832 * Preserve at least one pageblock unless memory pressure
2833 * is really high.
2834 */
2835 if (!force && zone->nr_reserved_highatomic <=
2836 pageblock_nr_pages)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002837 continue;
2838
2839 spin_lock_irqsave(&zone->lock, flags);
2840 for (order = 0; order < MAX_ORDER; order++) {
2841 struct free_area *area = &(zone->free_area[order]);
2842
Dan Williamsb03641a2019-05-14 15:41:32 -07002843 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
Geliang Tanga16601c2016-01-14 15:20:30 -08002844 if (!page)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002845 continue;
2846
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002847 /*
Minchan Kim4855e4a2016-12-12 16:42:08 -08002848 * In page freeing path, migratetype change is racy so
2849 * we can counter several free pages in a pageblock
Ingo Molnarf0953a12021-05-06 18:06:47 -07002850 * in this loop although we changed the pageblock type
Minchan Kim4855e4a2016-12-12 16:42:08 -08002851 * from highatomic to ac->migratetype. So we should
2852 * adjust the count once.
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002853 */
Xishi Qiua6ffdc02017-05-03 14:52:52 -07002854 if (is_migrate_highatomic_page(page)) {
Minchan Kim4855e4a2016-12-12 16:42:08 -08002855 /*
2856 * It should never happen but changes to
2857 * locking could inadvertently allow a per-cpu
2858 * drain to add pages to MIGRATE_HIGHATOMIC
2859 * while unreserving so be safe and watch for
2860 * underflows.
2861 */
2862 zone->nr_reserved_highatomic -= min(
2863 pageblock_nr_pages,
2864 zone->nr_reserved_highatomic);
2865 }
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002866
2867 /*
2868 * Convert to ac->migratetype and avoid the normal
2869 * pageblock stealing heuristics. Minimally, the caller
2870 * is doing the work and needs the pages. More
2871 * importantly, if the block was always converted to
2872 * MIGRATE_UNMOVABLE or another type then the number
2873 * of pageblocks that cannot be completely freed
2874 * may increase.
2875 */
2876 set_pageblock_migratetype(page, ac->migratetype);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002877 ret = move_freepages_block(zone, page, ac->migratetype,
2878 NULL);
Minchan Kim29fac032016-12-12 16:42:14 -08002879 if (ret) {
2880 spin_unlock_irqrestore(&zone->lock, flags);
2881 return ret;
2882 }
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002883 }
2884 spin_unlock_irqrestore(&zone->lock, flags);
2885 }
Minchan Kim04c87162016-12-12 16:42:11 -08002886
2887 return false;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002888}
2889
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002890/*
2891 * Try finding a free buddy page on the fallback list and put it on the free
2892 * list of requested migratetype, possibly along with other pages from the same
2893 * block, depending on fragmentation avoidance heuristics. Returns true if
2894 * fallback was found so that __rmqueue_smallest() can grab it.
Rasmus Villemoesb0025292017-07-10 15:49:26 -07002895 *
2896 * The use of signed ints for order and current_order is a deliberate
2897 * deviation from the rest of this file, to make the for loop
2898 * condition simpler.
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002899 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002900static __always_inline bool
Mel Gorman6bb15452018-12-28 00:35:41 -08002901__rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2902 unsigned int alloc_flags)
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002903{
Pintu Kumarb8af2942013-09-11 14:20:34 -07002904 struct free_area *area;
Rasmus Villemoesb0025292017-07-10 15:49:26 -07002905 int current_order;
Mel Gorman6bb15452018-12-28 00:35:41 -08002906 int min_order = order;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002907 struct page *page;
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002908 int fallback_mt;
2909 bool can_steal;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002910
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002911 /*
Mel Gorman6bb15452018-12-28 00:35:41 -08002912 * Do not steal pages from freelists belonging to other pageblocks
2913 * i.e. orders < pageblock_order. If there are no local zones free,
2914 * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2915 */
2916 if (alloc_flags & ALLOC_NOFRAGMENT)
2917 min_order = pageblock_order;
2918
2919 /*
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002920 * Find the largest available free page in the other list. This roughly
2921 * approximates finding the pageblock with the most free pages, which
2922 * would be too costly to do exactly.
2923 */
Mel Gorman6bb15452018-12-28 00:35:41 -08002924 for (current_order = MAX_ORDER - 1; current_order >= min_order;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002925 --current_order) {
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002926 area = &(zone->free_area[current_order]);
2927 fallback_mt = find_suitable_fallback(area, current_order,
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002928 start_migratetype, false, &can_steal);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002929 if (fallback_mt == -1)
2930 continue;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002931
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002932 /*
2933 * We cannot steal all free pages from the pageblock and the
2934 * requested migratetype is movable. In that case it's better to
2935 * steal and split the smallest available page instead of the
2936 * largest available page, because even if the next movable
2937 * allocation falls back into a different pageblock than this
2938 * one, it won't cause permanent fragmentation.
2939 */
2940 if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2941 && current_order > order)
2942 goto find_smallest;
Mel Gormane0104872007-10-16 01:25:53 -07002943
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002944 goto do_steal;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002945 }
2946
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002947 return false;
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002948
2949find_smallest:
2950 for (current_order = order; current_order < MAX_ORDER;
2951 current_order++) {
2952 area = &(zone->free_area[current_order]);
2953 fallback_mt = find_suitable_fallback(area, current_order,
2954 start_migratetype, false, &can_steal);
2955 if (fallback_mt != -1)
2956 break;
2957 }
2958
2959 /*
2960 * This should not happen - we already found a suitable fallback
2961 * when looking for the largest page.
2962 */
2963 VM_BUG_ON(current_order == MAX_ORDER);
2964
2965do_steal:
Dan Williamsb03641a2019-05-14 15:41:32 -07002966 page = get_page_from_free_area(area, fallback_mt);
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002967
Mel Gorman1c308442018-12-28 00:35:52 -08002968 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2969 can_steal);
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002970
2971 trace_mm_page_alloc_extfrag(page, order, current_order,
2972 start_migratetype, fallback_mt);
2973
2974 return true;
2975
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002976}
2977
Mel Gorman56fd56b2007-10-16 01:25:58 -07002978/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002979 * Do the hard work of removing an element from the buddy allocator.
2980 * Call me with the zone->lock already held.
2981 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002982static __always_inline struct page *
Mel Gorman6bb15452018-12-28 00:35:41 -08002983__rmqueue(struct zone *zone, unsigned int order, int migratetype,
2984 unsigned int alloc_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002985{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002986 struct page *page;
2987
Hailong liuce8f86e2021-01-12 15:49:08 -08002988 if (IS_ENABLED(CONFIG_CMA)) {
2989 /*
2990 * Balance movable allocations between regular and CMA areas by
2991 * allocating from CMA when over half of the zone's free memory
2992 * is in the CMA area.
2993 */
2994 if (alloc_flags & ALLOC_CMA &&
2995 zone_page_state(zone, NR_FREE_CMA_PAGES) >
2996 zone_page_state(zone, NR_FREE_PAGES) / 2) {
2997 page = __rmqueue_cma_fallback(zone, order);
2998 if (page)
2999 goto out;
3000 }
Roman Gushchin16867662020-06-03 15:58:42 -07003001 }
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07003002retry:
Mel Gorman56fd56b2007-10-16 01:25:58 -07003003 page = __rmqueue_smallest(zone, order, migratetype);
Mel Gorman974a7862015-11-06 16:28:34 -08003004 if (unlikely(!page)) {
Joonsoo Kim8510e692020-08-06 23:26:04 -07003005 if (alloc_flags & ALLOC_CMA)
Joonsoo Kimdc676472015-04-14 15:45:15 -07003006 page = __rmqueue_cma_fallback(zone, order);
3007
Mel Gorman6bb15452018-12-28 00:35:41 -08003008 if (!page && __rmqueue_fallback(zone, order, migratetype,
3009 alloc_flags))
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07003010 goto retry;
Mel Gorman728ec982009-06-16 15:32:04 -07003011 }
Hailong liuce8f86e2021-01-12 15:49:08 -08003012out:
3013 if (page)
3014 trace_mm_page_alloc_zone_locked(page, order, migratetype);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003015 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003016}
3017
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01003018/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003019 * Obtain a specified number of elements from the buddy allocator, all under
3020 * a single hold of the lock, for efficiency. Add them to the supplied list.
3021 * Returns the number of new pages which were placed at *list.
3022 */
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01003023static int rmqueue_bulk(struct zone *zone, unsigned int order,
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003024 unsigned long count, struct list_head *list,
Mel Gorman6bb15452018-12-28 00:35:41 -08003025 int migratetype, unsigned int alloc_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003026{
Mel Gormancb66bed2021-04-29 23:01:42 -07003027 int i, allocated = 0;
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01003028
Mel Gormandbbee9d2021-06-28 19:41:41 -07003029 /*
3030 * local_lock_irq held so equivalent to spin_lock_irqsave for
3031 * both PREEMPT_RT and non-PREEMPT_RT configurations.
3032 */
Mel Gormand34b0732017-04-20 14:37:43 -07003033 spin_lock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003034 for (i = 0; i < count; ++i) {
Mel Gorman6bb15452018-12-28 00:35:41 -08003035 struct page *page = __rmqueue(zone, order, migratetype,
3036 alloc_flags);
Nick Piggin085cc7d52006-01-06 00:11:01 -08003037 if (unlikely(page == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003038 break;
Mel Gorman81eabcb2007-12-17 16:20:05 -08003039
Mel Gorman479f8542016-05-19 17:14:35 -07003040 if (unlikely(check_pcp_refill(page)))
3041 continue;
3042
Mel Gorman81eabcb2007-12-17 16:20:05 -08003043 /*
Vlastimil Babka0fac3ba2017-11-15 17:38:07 -08003044 * Split buddy pages returned by expand() are received here in
3045 * physical page order. The page is added to the tail of
3046 * caller's list. From the callers perspective, the linked list
3047 * is ordered by page number under some conditions. This is
3048 * useful for IO devices that can forward direction from the
3049 * head, thus also in the physical page order. This is useful
3050 * for IO devices that can merge IO requests if the physical
3051 * pages are ordered properly.
Mel Gorman81eabcb2007-12-17 16:20:05 -08003052 */
Vlastimil Babka0fac3ba2017-11-15 17:38:07 -08003053 list_add_tail(&page->lru, list);
Mel Gormancb66bed2021-04-29 23:01:42 -07003054 allocated++;
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07003055 if (is_migrate_cma(get_pcppage_migratetype(page)))
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07003056 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
3057 -(1 << order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058 }
Mel Gormana6de7342016-12-12 16:44:41 -08003059
3060 /*
3061 * i pages were removed from the buddy list even if some leak due
3062 * to check_pcp_refill failing so adjust NR_FREE_PAGES based
Mel Gormancb66bed2021-04-29 23:01:42 -07003063 * on i. Do not confuse with 'allocated' which is the number of
Mel Gormana6de7342016-12-12 16:44:41 -08003064 * pages added to the pcp list.
3065 */
Mel Gormanf2260e62009-06-16 15:32:13 -07003066 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
Mel Gormand34b0732017-04-20 14:37:43 -07003067 spin_unlock(&zone->lock);
Mel Gormancb66bed2021-04-29 23:01:42 -07003068 return allocated;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003069}
3070
Christoph Lameter4ae7c032005-06-21 17:14:57 -07003071#ifdef CONFIG_NUMA
Christoph Lameter8fce4d82006-03-09 17:33:54 -08003072/*
Christoph Lameter4037d452007-05-09 02:35:14 -07003073 * Called from the vmstat counter updater to drain pagesets of this
3074 * currently executing processor on remote nodes after they have
3075 * expired.
3076 *
Christoph Lameter879336c2006-03-22 00:09:08 -08003077 * Note that this function must be called with the thread pinned to
3078 * a single processor.
Christoph Lameter8fce4d82006-03-09 17:33:54 -08003079 */
Christoph Lameter4037d452007-05-09 02:35:14 -07003080void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
Christoph Lameter4ae7c032005-06-21 17:14:57 -07003081{
Christoph Lameter4ae7c032005-06-21 17:14:57 -07003082 unsigned long flags;
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07003083 int to_drain, batch;
Christoph Lameter4ae7c032005-06-21 17:14:57 -07003084
Mel Gormandbbee9d2021-06-28 19:41:41 -07003085 local_lock_irqsave(&pagesets.lock, flags);
Jason Low4db0c3c2015-04-15 16:14:08 -07003086 batch = READ_ONCE(pcp->batch);
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07003087 to_drain = min(pcp->count, batch);
Aaron Lu77ba9062018-04-05 16:24:06 -07003088 if (to_drain > 0)
KOSAKI Motohiro2a135152012-07-31 16:42:53 -07003089 free_pcppages_bulk(zone, to_drain, pcp);
Mel Gormandbbee9d2021-06-28 19:41:41 -07003090 local_unlock_irqrestore(&pagesets.lock, flags);
Christoph Lameter4ae7c032005-06-21 17:14:57 -07003091}
3092#endif
3093
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003094/*
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003095 * Drain pcplists of the indicated processor and zone.
3096 *
3097 * The processor must either be the current processor and the
3098 * thread pinned to the current processor or a processor that
3099 * is not online.
3100 */
3101static void drain_pages_zone(unsigned int cpu, struct zone *zone)
3102{
3103 unsigned long flags;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003104 struct per_cpu_pages *pcp;
3105
Mel Gormandbbee9d2021-06-28 19:41:41 -07003106 local_lock_irqsave(&pagesets.lock, flags);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003107
Mel Gorman28f836b2021-06-28 19:41:38 -07003108 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
Aaron Lu77ba9062018-04-05 16:24:06 -07003109 if (pcp->count)
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003110 free_pcppages_bulk(zone, pcp->count, pcp);
Mel Gorman28f836b2021-06-28 19:41:38 -07003111
Mel Gormandbbee9d2021-06-28 19:41:41 -07003112 local_unlock_irqrestore(&pagesets.lock, flags);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003113}
3114
3115/*
3116 * Drain pcplists of all zones on the indicated processor.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003117 *
3118 * The processor must either be the current processor and the
3119 * thread pinned to the current processor or a processor that
3120 * is not online.
3121 */
3122static void drain_pages(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003123{
3124 struct zone *zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003125
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07003126 for_each_populated_zone(zone) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003127 drain_pages_zone(cpu, zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003128 }
3129}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003131/*
3132 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003133 *
3134 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
3135 * the single zone's pages.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003136 */
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003137void drain_local_pages(struct zone *zone)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003138{
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003139 int cpu = smp_processor_id();
3140
3141 if (zone)
3142 drain_pages_zone(cpu, zone);
3143 else
3144 drain_pages(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003145}
3146
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003147static void drain_local_pages_wq(struct work_struct *work)
3148{
Wei Yangd9367bd2018-12-28 00:38:58 -08003149 struct pcpu_drain *drain;
3150
3151 drain = container_of(work, struct pcpu_drain, work);
3152
Michal Hockoa459eeb2017-02-24 14:56:35 -08003153 /*
3154 * drain_all_pages doesn't use proper cpu hotplug protection so
3155 * we can race with cpu offline when the WQ can move this from
3156 * a cpu pinned worker to an unbound one. We can operate on a different
Ingo Molnarf0953a12021-05-06 18:06:47 -07003157 * cpu which is alright but we also have to make sure to not move to
Michal Hockoa459eeb2017-02-24 14:56:35 -08003158 * a different one.
3159 */
Sebastian Andrzej Siewior9c25cbf2021-11-05 13:40:52 -07003160 migrate_disable();
Wei Yangd9367bd2018-12-28 00:38:58 -08003161 drain_local_pages(drain->zone);
Sebastian Andrzej Siewior9c25cbf2021-11-05 13:40:52 -07003162 migrate_enable();
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003163}
3164
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003165/*
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08003166 * The implementation of drain_all_pages(), exposing an extra parameter to
3167 * drain on all cpus.
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003168 *
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08003169 * drain_all_pages() is optimized to only execute on cpus where pcplists are
3170 * not empty. The check for non-emptiness can however race with a free to
3171 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers
3172 * that need the guarantee that every CPU has drained can disable the
3173 * optimizing racy check.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003174 */
Zou Wei3b1f3652020-12-14 19:11:12 -08003175static void __drain_all_pages(struct zone *zone, bool force_all_cpus)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003176{
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003177 int cpu;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003178
3179 /*
Zhen Lei041711c2021-06-30 18:53:17 -07003180 * Allocate in the BSS so we won't require allocation in
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003181 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
3182 */
3183 static cpumask_t cpus_with_pcps;
3184
Michal Hockoce612872017-04-07 16:05:05 -07003185 /*
3186 * Make sure nobody triggers this path before mm_percpu_wq is fully
3187 * initialized.
3188 */
3189 if (WARN_ON_ONCE(!mm_percpu_wq))
3190 return;
3191
Mel Gormanbd233f52017-02-24 14:56:56 -08003192 /*
3193 * Do not drain if one is already in progress unless it's specific to
3194 * a zone. Such callers are primarily CMA and memory hotplug and need
3195 * the drain to be complete when the call returns.
3196 */
3197 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
3198 if (!zone)
3199 return;
3200 mutex_lock(&pcpu_drain_mutex);
3201 }
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003202
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003203 /*
3204 * We don't care about racing with CPU hotplug event
3205 * as offline notification will cause the notified
3206 * cpu to drain that CPU pcps and on_each_cpu_mask
3207 * disables preemption as part of its processing
3208 */
3209 for_each_online_cpu(cpu) {
Mel Gorman28f836b2021-06-28 19:41:38 -07003210 struct per_cpu_pages *pcp;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003211 struct zone *z;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003212 bool has_pcps = false;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003213
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08003214 if (force_all_cpus) {
3215 /*
3216 * The pcp.count check is racy, some callers need a
3217 * guarantee that no cpu is missed.
3218 */
3219 has_pcps = true;
3220 } else if (zone) {
Mel Gorman28f836b2021-06-28 19:41:38 -07003221 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
3222 if (pcp->count)
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003223 has_pcps = true;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003224 } else {
3225 for_each_populated_zone(z) {
Mel Gorman28f836b2021-06-28 19:41:38 -07003226 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu);
3227 if (pcp->count) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003228 has_pcps = true;
3229 break;
3230 }
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003231 }
3232 }
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003233
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003234 if (has_pcps)
3235 cpumask_set_cpu(cpu, &cpus_with_pcps);
3236 else
3237 cpumask_clear_cpu(cpu, &cpus_with_pcps);
3238 }
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003239
Mel Gormanbd233f52017-02-24 14:56:56 -08003240 for_each_cpu(cpu, &cpus_with_pcps) {
Wei Yangd9367bd2018-12-28 00:38:58 -08003241 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
3242
3243 drain->zone = zone;
3244 INIT_WORK(&drain->work, drain_local_pages_wq);
3245 queue_work_on(cpu, mm_percpu_wq, &drain->work);
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003246 }
Mel Gormanbd233f52017-02-24 14:56:56 -08003247 for_each_cpu(cpu, &cpus_with_pcps)
Wei Yangd9367bd2018-12-28 00:38:58 -08003248 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
Mel Gormanbd233f52017-02-24 14:56:56 -08003249
3250 mutex_unlock(&pcpu_drain_mutex);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003251}
3252
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08003253/*
3254 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
3255 *
3256 * When zone parameter is non-NULL, spill just the single zone's pages.
3257 *
3258 * Note that this can be extremely slow as the draining happens in a workqueue.
3259 */
3260void drain_all_pages(struct zone *zone)
3261{
3262 __drain_all_pages(zone, false);
3263}
3264
Rafael J. Wysocki296699d2007-07-29 23:27:18 +02003265#ifdef CONFIG_HIBERNATION
Linus Torvalds1da177e2005-04-16 15:20:36 -07003266
Chen Yu556b9692017-08-25 15:55:30 -07003267/*
3268 * Touch the watchdog for every WD_PAGE_COUNT pages.
3269 */
3270#define WD_PAGE_COUNT (128*1024)
3271
Linus Torvalds1da177e2005-04-16 15:20:36 -07003272void mark_free_pages(struct zone *zone)
3273{
Chen Yu556b9692017-08-25 15:55:30 -07003274 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003275 unsigned long flags;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003276 unsigned int order, t;
Geliang Tang86760a22016-01-14 15:20:33 -08003277 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003278
Xishi Qiu8080fc02013-09-11 14:21:45 -07003279 if (zone_is_empty(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003280 return;
3281
3282 spin_lock_irqsave(&zone->lock, flags);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003283
Cody P Schafer108bcc92013-02-22 16:35:23 -08003284 max_zone_pfn = zone_end_pfn(zone);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003285 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
3286 if (pfn_valid(pfn)) {
Geliang Tang86760a22016-01-14 15:20:33 -08003287 page = pfn_to_page(pfn);
Joonsoo Kimba6b0972016-05-19 17:12:16 -07003288
Chen Yu556b9692017-08-25 15:55:30 -07003289 if (!--page_count) {
3290 touch_nmi_watchdog();
3291 page_count = WD_PAGE_COUNT;
3292 }
3293
Joonsoo Kimba6b0972016-05-19 17:12:16 -07003294 if (page_zone(page) != zone)
3295 continue;
3296
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07003297 if (!swsusp_page_is_forbidden(page))
3298 swsusp_unset_page_free(page);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003299 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003300
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003301 for_each_migratetype_order(order, t) {
Geliang Tang86760a22016-01-14 15:20:33 -08003302 list_for_each_entry(page,
3303 &zone->free_area[order].free_list[t], lru) {
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003304 unsigned long i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003305
Geliang Tang86760a22016-01-14 15:20:33 -08003306 pfn = page_to_pfn(page);
Chen Yu556b9692017-08-25 15:55:30 -07003307 for (i = 0; i < (1UL << order); i++) {
3308 if (!--page_count) {
3309 touch_nmi_watchdog();
3310 page_count = WD_PAGE_COUNT;
3311 }
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07003312 swsusp_set_page_free(pfn_to_page(pfn + i));
Chen Yu556b9692017-08-25 15:55:30 -07003313 }
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003314 }
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003315 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003316 spin_unlock_irqrestore(&zone->lock, flags);
3317}
Mel Gormane2c55dc2007-10-16 01:25:50 -07003318#endif /* CONFIG_PM */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003319
Mel Gorman44042b42021-06-28 19:43:08 -07003320static bool free_unref_page_prepare(struct page *page, unsigned long pfn,
3321 unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003322{
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003323 int migratetype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003324
Mel Gorman44042b42021-06-28 19:43:08 -07003325 if (!free_pcp_prepare(page, order))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003326 return false;
Hugh Dickins689bceb2005-11-21 21:32:20 -08003327
Mel Gormandc4b0ca2014-06-04 16:10:17 -07003328 migratetype = get_pfnblock_migratetype(page, pfn);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07003329 set_pcppage_migratetype(page, migratetype);
Mel Gorman9cca35d42017-11-15 17:37:37 -08003330 return true;
3331}
3332
Mel Gorman3b12e7e2021-06-28 19:42:18 -07003333static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch)
3334{
3335 int min_nr_free, max_nr_free;
3336
3337 /* Check for PCP disabled or boot pageset */
3338 if (unlikely(high < batch))
3339 return 1;
3340
3341 /* Leave at least pcp->batch pages on the list */
3342 min_nr_free = batch;
3343 max_nr_free = high - batch;
3344
3345 /*
3346 * Double the number of pages freed each time there is subsequent
3347 * freeing of pages without any allocation.
3348 */
3349 batch <<= pcp->free_factor;
3350 if (batch < max_nr_free)
3351 pcp->free_factor++;
3352 batch = clamp(batch, min_nr_free, max_nr_free);
3353
3354 return batch;
3355}
3356
Mel Gormanc49c2c42021-06-28 19:42:21 -07003357static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone)
3358{
3359 int high = READ_ONCE(pcp->high);
3360
3361 if (unlikely(!high))
3362 return 0;
3363
3364 if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags))
3365 return high;
3366
3367 /*
3368 * If reclaim is active, limit the number of pages that can be
3369 * stored on pcp lists
3370 */
3371 return min(READ_ONCE(pcp->batch) << 2, high);
3372}
3373
Mel Gormandf1acc82021-06-28 19:42:00 -07003374static void free_unref_page_commit(struct page *page, unsigned long pfn,
Mel Gorman44042b42021-06-28 19:43:08 -07003375 int migratetype, unsigned int order)
Mel Gorman9cca35d42017-11-15 17:37:37 -08003376{
3377 struct zone *zone = page_zone(page);
3378 struct per_cpu_pages *pcp;
Mel Gorman3b12e7e2021-06-28 19:42:18 -07003379 int high;
Mel Gorman44042b42021-06-28 19:43:08 -07003380 int pindex;
Mel Gorman9cca35d42017-11-15 17:37:37 -08003381
Mel Gormand34b0732017-04-20 14:37:43 -07003382 __count_vm_event(PGFREE);
Mel Gorman28f836b2021-06-28 19:41:38 -07003383 pcp = this_cpu_ptr(zone->per_cpu_pageset);
Mel Gorman44042b42021-06-28 19:43:08 -07003384 pindex = order_to_pindex(migratetype, order);
3385 list_add(&page->lru, &pcp->lists[pindex]);
3386 pcp->count += 1 << order;
Mel Gormanc49c2c42021-06-28 19:42:21 -07003387 high = nr_pcp_high(pcp, zone);
Mel Gorman3b12e7e2021-06-28 19:42:18 -07003388 if (pcp->count >= high) {
3389 int batch = READ_ONCE(pcp->batch);
3390
3391 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp);
3392 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003393}
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003394
Mel Gorman9cca35d42017-11-15 17:37:37 -08003395/*
Mel Gorman44042b42021-06-28 19:43:08 -07003396 * Free a pcp page
Mel Gorman9cca35d42017-11-15 17:37:37 -08003397 */
Mel Gorman44042b42021-06-28 19:43:08 -07003398void free_unref_page(struct page *page, unsigned int order)
Mel Gorman9cca35d42017-11-15 17:37:37 -08003399{
3400 unsigned long flags;
3401 unsigned long pfn = page_to_pfn(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003402 int migratetype;
3403
Mel Gorman44042b42021-06-28 19:43:08 -07003404 if (!free_unref_page_prepare(page, pfn, order))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003405 return;
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -07003406
Nick Piggin9858db52006-10-11 01:21:30 -07003407 /*
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -07003408 * We only track unmovable, reclaimable and movable on pcp lists.
Mel Gormandf1acc82021-06-28 19:42:00 -07003409 * Place ISOLATE pages on the isolated list because they are being
Nick Piggindafb1362006-10-11 01:21:30 -07003410 * offlined but treat HIGHATOMIC as movable pages so we can get those
Hugh Dickins689bceb2005-11-21 21:32:20 -08003411 * areas back if necessary. Otherwise, we may have to free
3412 * excessively into the page allocator
Christoph Lameter3dfa5722008-02-04 22:29:19 -08003413 */
Mel Gormandf1acc82021-06-28 19:42:00 -07003414 migratetype = get_pcppage_migratetype(page);
3415 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003416 if (unlikely(is_migrate_isolate(migratetype))) {
Mel Gorman44042b42021-06-28 19:43:08 -07003417 free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE);
Christoph Lameterf8891e52006-06-30 01:55:45 -07003418 return;
Mel Gormanda456f12009-06-16 15:32:08 -07003419 }
Christoph Lameter3dfa5722008-02-04 22:29:19 -08003420 migratetype = MIGRATE_MOVABLE;
3421 }
3422
Mel Gormandbbee9d2021-06-28 19:41:41 -07003423 local_lock_irqsave(&pagesets.lock, flags);
Mel Gorman44042b42021-06-28 19:43:08 -07003424 free_unref_page_commit(page, pfn, migratetype, order);
Mel Gormandbbee9d2021-06-28 19:41:41 -07003425 local_unlock_irqrestore(&pagesets.lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003426}
3427
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003428/*
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003429 * Free a list of 0-order pages
3430 */
Mel Gorman2d4894b2017-11-15 17:37:59 -08003431void free_unref_page_list(struct list_head *list)
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003432{
3433 struct page *page, *next;
Mel Gorman9cca35d42017-11-15 17:37:37 -08003434 unsigned long flags, pfn;
Lucas Stachc24ad772017-12-14 15:32:55 -08003435 int batch_count = 0;
Mel Gormandf1acc82021-06-28 19:42:00 -07003436 int migratetype;
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003437
Mel Gorman9cca35d42017-11-15 17:37:37 -08003438 /* Prepare pages for freeing */
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003439 list_for_each_entry_safe(page, next, list, lru) {
Mel Gorman9cca35d42017-11-15 17:37:37 -08003440 pfn = page_to_pfn(page);
Miaohe Lin053cfda2021-09-08 18:10:11 -07003441 if (!free_unref_page_prepare(page, pfn, 0)) {
Mel Gorman9cca35d42017-11-15 17:37:37 -08003442 list_del(&page->lru);
Miaohe Lin053cfda2021-09-08 18:10:11 -07003443 continue;
3444 }
Mel Gormandf1acc82021-06-28 19:42:00 -07003445
3446 /*
3447 * Free isolated pages directly to the allocator, see
3448 * comment in free_unref_page.
3449 */
3450 migratetype = get_pcppage_migratetype(page);
Doug Berger47aef602021-08-19 19:04:12 -07003451 if (unlikely(is_migrate_isolate(migratetype))) {
3452 list_del(&page->lru);
3453 free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE);
3454 continue;
Mel Gormandf1acc82021-06-28 19:42:00 -07003455 }
3456
Mel Gorman9cca35d42017-11-15 17:37:37 -08003457 set_page_private(page, pfn);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003458 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003459
Mel Gormandbbee9d2021-06-28 19:41:41 -07003460 local_lock_irqsave(&pagesets.lock, flags);
Mel Gorman9cca35d42017-11-15 17:37:37 -08003461 list_for_each_entry_safe(page, next, list, lru) {
Mel Gormandf1acc82021-06-28 19:42:00 -07003462 pfn = page_private(page);
Mel Gorman9cca35d42017-11-15 17:37:37 -08003463 set_page_private(page, 0);
Doug Berger47aef602021-08-19 19:04:12 -07003464
3465 /*
3466 * Non-isolated types over MIGRATE_PCPTYPES get added
3467 * to the MIGRATE_MOVABLE pcp list.
3468 */
Mel Gormandf1acc82021-06-28 19:42:00 -07003469 migratetype = get_pcppage_migratetype(page);
Doug Berger47aef602021-08-19 19:04:12 -07003470 if (unlikely(migratetype >= MIGRATE_PCPTYPES))
3471 migratetype = MIGRATE_MOVABLE;
3472
Mel Gorman2d4894b2017-11-15 17:37:59 -08003473 trace_mm_page_free_batched(page);
Mel Gorman44042b42021-06-28 19:43:08 -07003474 free_unref_page_commit(page, pfn, migratetype, 0);
Lucas Stachc24ad772017-12-14 15:32:55 -08003475
3476 /*
3477 * Guard against excessive IRQ disabled times when we get
3478 * a large list of pages to free.
3479 */
3480 if (++batch_count == SWAP_CLUSTER_MAX) {
Mel Gormandbbee9d2021-06-28 19:41:41 -07003481 local_unlock_irqrestore(&pagesets.lock, flags);
Lucas Stachc24ad772017-12-14 15:32:55 -08003482 batch_count = 0;
Mel Gormandbbee9d2021-06-28 19:41:41 -07003483 local_lock_irqsave(&pagesets.lock, flags);
Lucas Stachc24ad772017-12-14 15:32:55 -08003484 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003485 }
Mel Gormandbbee9d2021-06-28 19:41:41 -07003486 local_unlock_irqrestore(&pagesets.lock, flags);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003487}
3488
3489/*
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003490 * split_page takes a non-compound higher-order page, and splits it into
3491 * n (1<<order) sub-pages: page[0..n]
3492 * Each sub-page must be freed individually.
3493 *
3494 * Note: this is probably too low level an operation for use in drivers.
3495 * Please consult with lkml before using this in your driver.
3496 */
3497void split_page(struct page *page, unsigned int order)
3498{
3499 int i;
3500
Sasha Levin309381fea2014-01-23 15:52:54 -08003501 VM_BUG_ON_PAGE(PageCompound(page), page);
3502 VM_BUG_ON_PAGE(!page_count(page), page);
Vegard Nossumb1eeab62008-11-25 16:55:53 +01003503
Joonsoo Kima9627bc2016-07-26 15:23:49 -07003504 for (i = 1; i < (1 << order); i++)
Nick Piggin7835e982006-03-22 00:08:40 -08003505 set_page_refcounted(page + i);
Matthew Wilcox (Oracle)8fb156c2020-10-15 20:05:29 -07003506 split_page_owner(page, 1 << order);
Zhou Guanghuie1baddf2021-03-12 21:08:33 -08003507 split_page_memcg(page, 1 << order);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003508}
K. Y. Srinivasan5853ff22013-03-25 15:47:38 -07003509EXPORT_SYMBOL_GPL(split_page);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003510
Joonsoo Kim3c605092014-11-13 15:19:21 -08003511int __isolate_free_page(struct page *page, unsigned int order)
Mel Gorman748446b2010-05-24 14:32:27 -07003512{
Mel Gorman748446b2010-05-24 14:32:27 -07003513 unsigned long watermark;
3514 struct zone *zone;
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07003515 int mt;
Mel Gorman748446b2010-05-24 14:32:27 -07003516
3517 BUG_ON(!PageBuddy(page));
3518
3519 zone = page_zone(page);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003520 mt = get_pageblock_migratetype(page);
Mel Gorman748446b2010-05-24 14:32:27 -07003521
Minchan Kim194159f2013-02-22 16:33:58 -08003522 if (!is_migrate_isolate(mt)) {
Vlastimil Babka8348faf2016-10-07 16:58:00 -07003523 /*
3524 * Obey watermarks as if the page was being allocated. We can
3525 * emulate a high-order watermark check with a raised order-0
3526 * watermark, because we already know our high-order page
3527 * exists.
3528 */
Mel Gormanfd1444b2019-03-05 15:44:50 -08003529 watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003530 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003531 return 0;
3532
Mel Gorman8fb74b92013-01-11 14:32:16 -08003533 __mod_zone_freepage_state(zone, -(1UL << order), mt);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003534 }
Mel Gorman748446b2010-05-24 14:32:27 -07003535
3536 /* Remove page from free list */
Dan Williamsb03641a2019-05-14 15:41:32 -07003537
Alexander Duyck6ab01362020-04-06 20:04:49 -07003538 del_page_from_free_list(page, zone, order);
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07003539
zhong jiang400bc7f2016-07-28 15:45:07 -07003540 /*
3541 * Set the pageblock if the isolated page is at least half of a
3542 * pageblock
3543 */
Mel Gorman748446b2010-05-24 14:32:27 -07003544 if (order >= pageblock_order - 1) {
3545 struct page *endpage = page + (1 << order) - 1;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01003546 for (; page < endpage; page += pageblock_nr_pages) {
3547 int mt = get_pageblock_migratetype(page);
Minchan Kim88ed3652016-12-12 16:42:05 -08003548 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
Xishi Qiua6ffdc02017-05-03 14:52:52 -07003549 && !is_migrate_highatomic(mt))
Michal Nazarewicz47118af2011-12-29 13:09:50 +01003550 set_pageblock_migratetype(page,
3551 MIGRATE_MOVABLE);
3552 }
Mel Gorman748446b2010-05-24 14:32:27 -07003553 }
3554
Joonsoo Kimf3a14ce2015-07-17 16:24:15 -07003555
Mel Gorman8fb74b92013-01-11 14:32:16 -08003556 return 1UL << order;
Mel Gorman1fb3f8c2012-10-08 16:29:12 -07003557}
3558
Alexander Duyck624f58d2020-04-06 20:04:53 -07003559/**
3560 * __putback_isolated_page - Return a now-isolated page back where we got it
3561 * @page: Page that was isolated
3562 * @order: Order of the isolated page
Randy Dunlape6a0a7a2020-04-10 14:32:29 -07003563 * @mt: The page's pageblock's migratetype
Alexander Duyck624f58d2020-04-06 20:04:53 -07003564 *
3565 * This function is meant to return a page pulled from the free lists via
3566 * __isolate_free_page back to the free lists they were pulled from.
3567 */
3568void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3569{
3570 struct zone *zone = page_zone(page);
3571
3572 /* zone lock should be held when this function is called */
3573 lockdep_assert_held(&zone->lock);
3574
3575 /* Return isolated page to tail of freelist. */
David Hildenbrandf04a5d52020-10-15 20:09:20 -07003576 __free_one_page(page, page_to_pfn(page), zone, order, mt,
David Hildenbrand47b6a24a22020-10-15 20:09:26 -07003577 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
Alexander Duyck624f58d2020-04-06 20:04:53 -07003578}
3579
Mel Gorman1fb3f8c2012-10-08 16:29:12 -07003580/*
Mel Gorman060e7412016-05-19 17:13:27 -07003581 * Update NUMA hit/miss statistics
3582 *
3583 * Must be called with interrupts disabled.
Mel Gorman060e7412016-05-19 17:13:27 -07003584 */
Mel Gorman3e230602021-06-28 19:41:50 -07003585static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
3586 long nr_account)
Mel Gorman060e7412016-05-19 17:13:27 -07003587{
3588#ifdef CONFIG_NUMA
Kemi Wang3a321d22017-09-08 16:12:48 -07003589 enum numa_stat_item local_stat = NUMA_LOCAL;
Mel Gorman060e7412016-05-19 17:13:27 -07003590
Kemi Wang45180852017-11-15 17:38:22 -08003591 /* skip numa counters update if numa stats is disabled */
3592 if (!static_branch_likely(&vm_numa_stat_key))
3593 return;
3594
Pavel Tatashinc1093b72018-08-21 21:53:32 -07003595 if (zone_to_nid(z) != numa_node_id())
Mel Gorman060e7412016-05-19 17:13:27 -07003596 local_stat = NUMA_OTHER;
Mel Gorman060e7412016-05-19 17:13:27 -07003597
Pavel Tatashinc1093b72018-08-21 21:53:32 -07003598 if (zone_to_nid(z) == zone_to_nid(preferred_zone))
Mel Gorman3e230602021-06-28 19:41:50 -07003599 __count_numa_events(z, NUMA_HIT, nr_account);
Michal Hocko2df26632017-01-10 16:57:39 -08003600 else {
Mel Gorman3e230602021-06-28 19:41:50 -07003601 __count_numa_events(z, NUMA_MISS, nr_account);
3602 __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account);
Mel Gorman060e7412016-05-19 17:13:27 -07003603 }
Mel Gorman3e230602021-06-28 19:41:50 -07003604 __count_numa_events(z, local_stat, nr_account);
Mel Gorman060e7412016-05-19 17:13:27 -07003605#endif
3606}
3607
Mel Gorman066b2392017-02-24 14:56:26 -08003608/* Remove page from the per-cpu list, caller must protect the list */
Jesper Dangaard Brouer3b822012021-04-29 23:01:55 -07003609static inline
Mel Gorman44042b42021-06-28 19:43:08 -07003610struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
3611 int migratetype,
Mel Gorman6bb15452018-12-28 00:35:41 -08003612 unsigned int alloc_flags,
Mel Gorman453f85d2017-11-15 17:38:03 -08003613 struct per_cpu_pages *pcp,
Mel Gorman066b2392017-02-24 14:56:26 -08003614 struct list_head *list)
3615{
3616 struct page *page;
3617
3618 do {
3619 if (list_empty(list)) {
Mel Gorman44042b42021-06-28 19:43:08 -07003620 int batch = READ_ONCE(pcp->batch);
3621 int alloced;
3622
3623 /*
3624 * Scale batch relative to order if batch implies
3625 * free pages can be stored on the PCP. Batch can
3626 * be 1 for small zones or for boot pagesets which
3627 * should never store free pages as the pages may
3628 * belong to arbitrary zones.
3629 */
3630 if (batch > 1)
3631 batch = max(batch >> order, 2);
3632 alloced = rmqueue_bulk(zone, order,
3633 batch, list,
Mel Gorman6bb15452018-12-28 00:35:41 -08003634 migratetype, alloc_flags);
Mel Gorman44042b42021-06-28 19:43:08 -07003635
3636 pcp->count += alloced << order;
Mel Gorman066b2392017-02-24 14:56:26 -08003637 if (unlikely(list_empty(list)))
3638 return NULL;
3639 }
3640
Mel Gorman453f85d2017-11-15 17:38:03 -08003641 page = list_first_entry(list, struct page, lru);
Mel Gorman066b2392017-02-24 14:56:26 -08003642 list_del(&page->lru);
Mel Gorman44042b42021-06-28 19:43:08 -07003643 pcp->count -= 1 << order;
Mel Gorman066b2392017-02-24 14:56:26 -08003644 } while (check_new_pcp(page));
3645
3646 return page;
3647}
3648
3649/* Lock and remove page from the per-cpu list */
3650static struct page *rmqueue_pcplist(struct zone *preferred_zone,
Mel Gorman44042b42021-06-28 19:43:08 -07003651 struct zone *zone, unsigned int order,
3652 gfp_t gfp_flags, int migratetype,
3653 unsigned int alloc_flags)
Mel Gorman066b2392017-02-24 14:56:26 -08003654{
3655 struct per_cpu_pages *pcp;
3656 struct list_head *list;
Mel Gorman066b2392017-02-24 14:56:26 -08003657 struct page *page;
Mel Gormand34b0732017-04-20 14:37:43 -07003658 unsigned long flags;
Mel Gorman066b2392017-02-24 14:56:26 -08003659
Mel Gormandbbee9d2021-06-28 19:41:41 -07003660 local_lock_irqsave(&pagesets.lock, flags);
Mel Gorman3b12e7e2021-06-28 19:42:18 -07003661
3662 /*
3663 * On allocation, reduce the number of pages that are batch freed.
3664 * See nr_pcp_free() where free_factor is increased for subsequent
3665 * frees.
3666 */
Mel Gorman28f836b2021-06-28 19:41:38 -07003667 pcp = this_cpu_ptr(zone->per_cpu_pageset);
Mel Gorman3b12e7e2021-06-28 19:42:18 -07003668 pcp->free_factor >>= 1;
Mel Gorman44042b42021-06-28 19:43:08 -07003669 list = &pcp->lists[order_to_pindex(migratetype, order)];
3670 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list);
Mel Gorman43c95bc2021-06-28 19:41:54 -07003671 local_unlock_irqrestore(&pagesets.lock, flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003672 if (page) {
Yafang Shao1c52e6d2019-05-13 17:22:40 -07003673 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
Mel Gorman3e230602021-06-28 19:41:50 -07003674 zone_statistics(preferred_zone, zone, 1);
Mel Gorman066b2392017-02-24 14:56:26 -08003675 }
Mel Gorman066b2392017-02-24 14:56:26 -08003676 return page;
3677}
3678
Mel Gorman060e7412016-05-19 17:13:27 -07003679/*
Vlastimil Babka75379192015-02-11 15:25:38 -08003680 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003681 */
Mel Gorman0a15c3e2009-06-16 15:32:05 -07003682static inline
Mel Gorman066b2392017-02-24 14:56:26 -08003683struct page *rmqueue(struct zone *preferred_zone,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003684 struct zone *zone, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07003685 gfp_t gfp_flags, unsigned int alloc_flags,
3686 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003687{
3688 unsigned long flags;
Hugh Dickins689bceb2005-11-21 21:32:20 -08003689 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003690
Mel Gorman44042b42021-06-28 19:43:08 -07003691 if (likely(pcp_allowed_order(order))) {
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003692 /*
3693 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and
3694 * we need to skip it when CMA area isn't allowed.
3695 */
3696 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
3697 migratetype != MIGRATE_MOVABLE) {
Mel Gorman44042b42021-06-28 19:43:08 -07003698 page = rmqueue_pcplist(preferred_zone, zone, order,
3699 gfp_flags, migratetype, alloc_flags);
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003700 goto out;
3701 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003702 }
3703
Mel Gorman066b2392017-02-24 14:56:26 -08003704 /*
3705 * We most definitely don't want callers attempting to
3706 * allocate greater than order-1 page units with __GFP_NOFAIL.
3707 */
3708 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3709 spin_lock_irqsave(&zone->lock, flags);
3710
3711 do {
3712 page = NULL;
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003713 /*
3714 * order-0 request can reach here when the pcplist is skipped
3715 * due to non-CMA allocation context. HIGHATOMIC area is
3716 * reserved for high-order atomic allocation, so order-0
3717 * request should skip it.
3718 */
3719 if (order > 0 && alloc_flags & ALLOC_HARDER) {
Mel Gorman066b2392017-02-24 14:56:26 -08003720 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3721 if (page)
3722 trace_mm_page_alloc_zone_locked(page, order, migratetype);
3723 }
3724 if (!page)
Mel Gorman6bb15452018-12-28 00:35:41 -08003725 page = __rmqueue(zone, order, migratetype, alloc_flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003726 } while (page && check_new_pages(page, order));
Mel Gorman066b2392017-02-24 14:56:26 -08003727 if (!page)
3728 goto failed;
Mel Gorman43c95bc2021-06-28 19:41:54 -07003729
Mel Gorman066b2392017-02-24 14:56:26 -08003730 __mod_zone_freepage_state(zone, -(1 << order),
3731 get_pcppage_migratetype(page));
Mel Gorman43c95bc2021-06-28 19:41:54 -07003732 spin_unlock_irqrestore(&zone->lock, flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003733
Mel Gorman16709d12016-07-28 15:46:56 -07003734 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
Mel Gorman3e230602021-06-28 19:41:50 -07003735 zone_statistics(preferred_zone, zone, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003736
Mel Gorman066b2392017-02-24 14:56:26 -08003737out:
Mel Gorman73444bc2019-01-08 15:23:39 -08003738 /* Separate test+clear to avoid unnecessary atomics */
3739 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3740 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3741 wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3742 }
3743
Mel Gorman066b2392017-02-24 14:56:26 -08003744 VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003745 return page;
Nick Piggina74609f2006-01-06 00:11:20 -08003746
3747failed:
Mel Gorman43c95bc2021-06-28 19:41:54 -07003748 spin_unlock_irqrestore(&zone->lock, flags);
Nick Piggina74609f2006-01-06 00:11:20 -08003749 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003750}
3751
Akinobu Mita933e3122006-12-08 02:39:45 -08003752#ifdef CONFIG_FAIL_PAGE_ALLOC
3753
Akinobu Mitab2588c42011-07-26 16:09:03 -07003754static struct {
Akinobu Mita933e3122006-12-08 02:39:45 -08003755 struct fault_attr attr;
3756
Viresh Kumar621a5f72015-09-26 15:04:07 -07003757 bool ignore_gfp_highmem;
Mel Gorman71baba42015-11-06 16:28:28 -08003758 bool ignore_gfp_reclaim;
Akinobu Mita54114992007-07-15 23:40:23 -07003759 u32 min_order;
Akinobu Mita933e3122006-12-08 02:39:45 -08003760} fail_page_alloc = {
3761 .attr = FAULT_ATTR_INITIALIZER,
Mel Gorman71baba42015-11-06 16:28:28 -08003762 .ignore_gfp_reclaim = true,
Viresh Kumar621a5f72015-09-26 15:04:07 -07003763 .ignore_gfp_highmem = true,
Akinobu Mita54114992007-07-15 23:40:23 -07003764 .min_order = 1,
Akinobu Mita933e3122006-12-08 02:39:45 -08003765};
3766
3767static int __init setup_fail_page_alloc(char *str)
3768{
3769 return setup_fault_attr(&fail_page_alloc.attr, str);
3770}
3771__setup("fail_page_alloc=", setup_fail_page_alloc);
3772
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003773static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08003774{
Akinobu Mita54114992007-07-15 23:40:23 -07003775 if (order < fail_page_alloc.min_order)
Gavin Shandeaf3862012-07-31 16:41:51 -07003776 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003777 if (gfp_mask & __GFP_NOFAIL)
Gavin Shandeaf3862012-07-31 16:41:51 -07003778 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003779 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
Gavin Shandeaf3862012-07-31 16:41:51 -07003780 return false;
Mel Gorman71baba42015-11-06 16:28:28 -08003781 if (fail_page_alloc.ignore_gfp_reclaim &&
3782 (gfp_mask & __GFP_DIRECT_RECLAIM))
Gavin Shandeaf3862012-07-31 16:41:51 -07003783 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003784
3785 return should_fail(&fail_page_alloc.attr, 1 << order);
3786}
3787
3788#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3789
3790static int __init fail_page_alloc_debugfs(void)
3791{
Joe Perches0825a6f2018-06-14 15:27:58 -07003792 umode_t mode = S_IFREG | 0600;
Akinobu Mita933e3122006-12-08 02:39:45 -08003793 struct dentry *dir;
Akinobu Mita933e3122006-12-08 02:39:45 -08003794
Akinobu Mitadd48c082011-08-03 16:21:01 -07003795 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3796 &fail_page_alloc.attr);
Akinobu Mita933e3122006-12-08 02:39:45 -08003797
Greg Kroah-Hartmand9f79792019-03-05 15:46:09 -08003798 debugfs_create_bool("ignore-gfp-wait", mode, dir,
3799 &fail_page_alloc.ignore_gfp_reclaim);
3800 debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3801 &fail_page_alloc.ignore_gfp_highmem);
3802 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
Akinobu Mita933e3122006-12-08 02:39:45 -08003803
Akinobu Mitab2588c42011-07-26 16:09:03 -07003804 return 0;
Akinobu Mita933e3122006-12-08 02:39:45 -08003805}
3806
3807late_initcall(fail_page_alloc_debugfs);
3808
3809#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3810
3811#else /* CONFIG_FAIL_PAGE_ALLOC */
3812
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003813static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08003814{
Gavin Shandeaf3862012-07-31 16:41:51 -07003815 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003816}
3817
3818#endif /* CONFIG_FAIL_PAGE_ALLOC */
3819
Matteo Croce54aa3862021-07-14 21:26:43 -07003820noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003821{
3822 return __should_fail_alloc_page(gfp_mask, order);
3823}
3824ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3825
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003826static inline long __zone_watermark_unusable_free(struct zone *z,
3827 unsigned int order, unsigned int alloc_flags)
3828{
3829 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3830 long unusable_free = (1 << order) - 1;
3831
3832 /*
3833 * If the caller does not have rights to ALLOC_HARDER then subtract
3834 * the high-atomic reserves. This will over-estimate the size of the
3835 * atomic reserve but it avoids a search.
3836 */
3837 if (likely(!alloc_harder))
3838 unusable_free += z->nr_reserved_highatomic;
3839
3840#ifdef CONFIG_CMA
3841 /* If allocation can't use CMA areas don't use free CMA pages */
3842 if (!(alloc_flags & ALLOC_CMA))
3843 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
3844#endif
3845
3846 return unusable_free;
3847}
3848
Linus Torvalds1da177e2005-04-16 15:20:36 -07003849/*
Mel Gorman97a16fc2015-11-06 16:28:40 -08003850 * Return true if free base pages are above 'mark'. For high-order checks it
3851 * will return true of the order-0 watermark is reached and there is at least
3852 * one free page of a suitable size. Checking now avoids taking the zone lock
3853 * to check in the allocation paths if no pages are free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003854 */
Michal Hocko86a294a2016-05-20 16:57:12 -07003855bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003856 int highest_zoneidx, unsigned int alloc_flags,
Michal Hocko86a294a2016-05-20 16:57:12 -07003857 long free_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003858{
Christoph Lameterd23ad422007-02-10 01:43:02 -08003859 long min = mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003860 int o;
Michal Hockocd04ae12017-09-06 16:24:50 -07003861 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003862
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003863 /* free_pages may go negative - that's OK */
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003864 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003865
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003866 if (alloc_flags & ALLOC_HIGH)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003867 min -= min / 2;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003868
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003869 if (unlikely(alloc_harder)) {
Michal Hockocd04ae12017-09-06 16:24:50 -07003870 /*
3871 * OOM victims can try even harder than normal ALLOC_HARDER
3872 * users on the grounds that it's definitely going to be in
3873 * the exit path shortly and free memory. Any allocation it
3874 * makes during the free path will be small and short-lived.
3875 */
3876 if (alloc_flags & ALLOC_OOM)
3877 min -= min / 2;
3878 else
3879 min -= min / 4;
3880 }
3881
Mel Gorman97a16fc2015-11-06 16:28:40 -08003882 /*
3883 * Check watermarks for an order-0 allocation request. If these
3884 * are not met, then a high-order request also cannot go ahead
3885 * even if a suitable page happened to be free.
3886 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003887 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
Mel Gorman88f5acf2011-01-13 15:45:41 -08003888 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003889
Mel Gorman97a16fc2015-11-06 16:28:40 -08003890 /* If this is an order-0 request then the watermark is fine */
3891 if (!order)
3892 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003893
Mel Gorman97a16fc2015-11-06 16:28:40 -08003894 /* For a high-order request, check at least one suitable page is free */
3895 for (o = order; o < MAX_ORDER; o++) {
3896 struct free_area *area = &z->free_area[o];
3897 int mt;
3898
3899 if (!area->nr_free)
3900 continue;
3901
Mel Gorman97a16fc2015-11-06 16:28:40 -08003902 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
Dan Williamsb03641a2019-05-14 15:41:32 -07003903 if (!free_area_empty(area, mt))
Mel Gorman97a16fc2015-11-06 16:28:40 -08003904 return true;
3905 }
3906
3907#ifdef CONFIG_CMA
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003908 if ((alloc_flags & ALLOC_CMA) &&
Dan Williamsb03641a2019-05-14 15:41:32 -07003909 !free_area_empty(area, MIGRATE_CMA)) {
Mel Gorman97a16fc2015-11-06 16:28:40 -08003910 return true;
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003911 }
Mel Gorman97a16fc2015-11-06 16:28:40 -08003912#endif
chenqiwu76089d02020-04-01 21:09:50 -07003913 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))
Vlastimil Babkab050e372017-11-15 17:38:30 -08003914 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003915 }
Mel Gorman97a16fc2015-11-06 16:28:40 -08003916 return false;
Mel Gorman88f5acf2011-01-13 15:45:41 -08003917}
3918
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003919bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003920 int highest_zoneidx, unsigned int alloc_flags)
Mel Gorman88f5acf2011-01-13 15:45:41 -08003921{
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003922 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
Mel Gorman88f5acf2011-01-13 15:45:41 -08003923 zone_page_state(z, NR_FREE_PAGES));
3924}
3925
Mel Gorman48ee5f32016-05-19 17:14:07 -07003926static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003927 unsigned long mark, int highest_zoneidx,
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003928 unsigned int alloc_flags, gfp_t gfp_mask)
Mel Gorman48ee5f32016-05-19 17:14:07 -07003929{
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003930 long free_pages;
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003931
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003932 free_pages = zone_page_state(z, NR_FREE_PAGES);
Mel Gorman48ee5f32016-05-19 17:14:07 -07003933
3934 /*
3935 * Fast check for order-0 only. If this fails then the reserves
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003936 * need to be calculated.
Mel Gorman48ee5f32016-05-19 17:14:07 -07003937 */
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003938 if (!order) {
3939 long fast_free;
3940
3941 fast_free = free_pages;
3942 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags);
3943 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx])
3944 return true;
3945 }
Mel Gorman48ee5f32016-05-19 17:14:07 -07003946
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003947 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3948 free_pages))
3949 return true;
3950 /*
3951 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations
3952 * when checking the min watermark. The min watermark is the
3953 * point where boosting is ignored so that kswapd is woken up
3954 * when below the low watermark.
3955 */
3956 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost
3957 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
3958 mark = z->_watermark[WMARK_MIN];
3959 return __zone_watermark_ok(z, order, mark, highest_zoneidx,
3960 alloc_flags, free_pages);
3961 }
3962
3963 return false;
Mel Gorman48ee5f32016-05-19 17:14:07 -07003964}
3965
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003966bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003967 unsigned long mark, int highest_zoneidx)
Mel Gorman88f5acf2011-01-13 15:45:41 -08003968{
3969 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3970
3971 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3972 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3973
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003974 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,
Mel Gorman88f5acf2011-01-13 15:45:41 -08003975 free_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003976}
3977
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003978#ifdef CONFIG_NUMA
Geert Uytterhoeven61bb6cd2021-11-05 13:40:24 -07003979int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE;
3980
David Rientjes957f8222012-10-08 16:33:24 -07003981static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3982{
Gavin Shane02dc012017-02-24 14:59:33 -08003983 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
Matt Fleminga55c7452019-08-08 20:53:01 +01003984 node_reclaim_distance;
David Rientjes957f8222012-10-08 16:33:24 -07003985}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003986#else /* CONFIG_NUMA */
David Rientjes957f8222012-10-08 16:33:24 -07003987static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3988{
3989 return true;
3990}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003991#endif /* CONFIG_NUMA */
3992
Mel Gorman6bb15452018-12-28 00:35:41 -08003993/*
3994 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3995 * fragmentation is subtle. If the preferred zone was HIGHMEM then
3996 * premature use of a lower zone may cause lowmem pressure problems that
3997 * are worse than fragmentation. If the next zone is ZONE_DMA then it is
3998 * probably too small. It only makes sense to spread allocations to avoid
3999 * fragmentation between the Normal and DMA32 zones.
4000 */
4001static inline unsigned int
Mel Gorman0a79cda2018-12-28 00:35:48 -08004002alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
Mel Gorman6bb15452018-12-28 00:35:41 -08004003{
Mateusz Nosek736838e2020-04-01 21:09:47 -07004004 unsigned int alloc_flags;
Mel Gorman0a79cda2018-12-28 00:35:48 -08004005
Mateusz Nosek736838e2020-04-01 21:09:47 -07004006 /*
4007 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4008 * to save a branch.
4009 */
4010 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
Mel Gorman0a79cda2018-12-28 00:35:48 -08004011
4012#ifdef CONFIG_ZONE_DMA32
Andrey Ryabinin8139ad02019-04-25 22:23:58 -07004013 if (!zone)
4014 return alloc_flags;
4015
Mel Gorman6bb15452018-12-28 00:35:41 -08004016 if (zone_idx(zone) != ZONE_NORMAL)
Andrey Ryabinin8118b822019-04-25 22:24:01 -07004017 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08004018
4019 /*
4020 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
4021 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
4022 * on UMA that if Normal is populated then so is DMA32.
4023 */
4024 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
4025 if (nr_online_nodes > 1 && !populated_zone(--zone))
Andrey Ryabinin8118b822019-04-25 22:24:01 -07004026 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08004027
Andrey Ryabinin8118b822019-04-25 22:24:01 -07004028 alloc_flags |= ALLOC_NOFRAGMENT;
Mel Gorman0a79cda2018-12-28 00:35:48 -08004029#endif /* CONFIG_ZONE_DMA32 */
4030 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08004031}
Mel Gorman6bb15452018-12-28 00:35:41 -08004032
Pavel Tatashin8e3560d2021-05-04 18:39:00 -07004033/* Must be called after current_gfp_context() which can change gfp_mask */
4034static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask,
4035 unsigned int alloc_flags)
Joonsoo Kim8510e692020-08-06 23:26:04 -07004036{
4037#ifdef CONFIG_CMA
Pavel Tatashin8e3560d2021-05-04 18:39:00 -07004038 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
Joonsoo Kim8510e692020-08-06 23:26:04 -07004039 alloc_flags |= ALLOC_CMA;
Joonsoo Kim8510e692020-08-06 23:26:04 -07004040#endif
4041 return alloc_flags;
4042}
4043
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004044/*
Paul Jackson0798e512006-12-06 20:31:38 -08004045 * get_page_from_freelist goes through the zonelist trying to allocate
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004046 * a page.
4047 */
4048static struct page *
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004049get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
4050 const struct alloc_context *ac)
Martin Hicks753ee722005-06-21 17:14:41 -07004051{
Mel Gorman6bb15452018-12-28 00:35:41 -08004052 struct zoneref *z;
Mel Gorman5117f452009-06-16 15:31:59 -07004053 struct zone *zone;
Mel Gorman3b8c0be2016-07-28 15:46:53 -07004054 struct pglist_data *last_pgdat_dirty_limit = NULL;
Mel Gorman6bb15452018-12-28 00:35:41 -08004055 bool no_fallback;
Mel Gorman3b8c0be2016-07-28 15:46:53 -07004056
Mel Gorman6bb15452018-12-28 00:35:41 -08004057retry:
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004058 /*
Paul Jackson9276b1bc2006-12-06 20:31:48 -08004059 * Scan zonelist, looking for a zone with enough free.
Vladimir Davydov344736f2014-10-20 15:50:30 +04004060 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004061 */
Mel Gorman6bb15452018-12-28 00:35:41 -08004062 no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
4063 z = ac->preferred_zoneref;
Mateusz Nosek30d8ec72020-10-13 16:55:57 -07004064 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
4065 ac->nodemask) {
Mel Gormanbe06af02016-05-19 17:13:47 -07004066 struct page *page;
Johannes Weinere085dbc2013-09-11 14:20:46 -07004067 unsigned long mark;
4068
Mel Gorman664eedd2014-06-04 16:10:08 -07004069 if (cpusets_enabled() &&
4070 (alloc_flags & ALLOC_CPUSET) &&
Vlastimil Babka002f2902016-05-19 17:14:30 -07004071 !__cpuset_zone_allowed(zone, gfp_mask))
Mel Gormancd38b112011-07-25 17:12:29 -07004072 continue;
Johannes Weinera756cf52012-01-10 15:07:49 -08004073 /*
4074 * When allocating a page cache page for writing, we
Mel Gorman281e3722016-07-28 15:46:11 -07004075 * want to get it from a node that is within its dirty
4076 * limit, such that no single node holds more than its
Johannes Weinera756cf52012-01-10 15:07:49 -08004077 * proportional share of globally allowed dirty pages.
Mel Gorman281e3722016-07-28 15:46:11 -07004078 * The dirty limits take into account the node's
Johannes Weinera756cf52012-01-10 15:07:49 -08004079 * lowmem reserves and high watermark so that kswapd
4080 * should be able to balance it without having to
4081 * write pages from its LRU list.
4082 *
Johannes Weinera756cf52012-01-10 15:07:49 -08004083 * XXX: For now, allow allocations to potentially
Mel Gorman281e3722016-07-28 15:46:11 -07004084 * exceed the per-node dirty limit in the slowpath
Mel Gormanc9ab0c42015-11-06 16:28:12 -08004085 * (spread_dirty_pages unset) before going into reclaim,
Johannes Weinera756cf52012-01-10 15:07:49 -08004086 * which is important when on a NUMA setup the allowed
Mel Gorman281e3722016-07-28 15:46:11 -07004087 * nodes are together not big enough to reach the
Johannes Weinera756cf52012-01-10 15:07:49 -08004088 * global limit. The proper fix for these situations
Mel Gorman281e3722016-07-28 15:46:11 -07004089 * will require awareness of nodes in the
Johannes Weinera756cf52012-01-10 15:07:49 -08004090 * dirty-throttling and the flusher threads.
4091 */
Mel Gorman3b8c0be2016-07-28 15:46:53 -07004092 if (ac->spread_dirty_pages) {
4093 if (last_pgdat_dirty_limit == zone->zone_pgdat)
4094 continue;
4095
4096 if (!node_dirty_ok(zone->zone_pgdat)) {
4097 last_pgdat_dirty_limit = zone->zone_pgdat;
4098 continue;
4099 }
4100 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004101
Mel Gorman6bb15452018-12-28 00:35:41 -08004102 if (no_fallback && nr_online_nodes > 1 &&
4103 zone != ac->preferred_zoneref->zone) {
4104 int local_nid;
4105
4106 /*
4107 * If moving to a remote node, retry but allow
4108 * fragmenting fallbacks. Locality is more important
4109 * than fragmentation avoidance.
4110 */
4111 local_nid = zone_to_nid(ac->preferred_zoneref->zone);
4112 if (zone_to_nid(zone) != local_nid) {
4113 alloc_flags &= ~ALLOC_NOFRAGMENT;
4114 goto retry;
4115 }
4116 }
4117
Mel Gormana9214442018-12-28 00:35:44 -08004118 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
Mel Gorman48ee5f32016-05-19 17:14:07 -07004119 if (!zone_watermark_fast(zone, order, mark,
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07004120 ac->highest_zoneidx, alloc_flags,
4121 gfp_mask)) {
Mel Gormanfa5e0842009-06-16 15:33:22 -07004122 int ret;
4123
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07004124#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
4125 /*
4126 * Watermark failed for this zone, but see if we can
4127 * grow this zone if it contains deferred pages.
4128 */
4129 if (static_branch_unlikely(&deferred_pages)) {
4130 if (_deferred_grow_zone(zone, order))
4131 goto try_this_zone;
4132 }
4133#endif
Mel Gorman5dab2912014-06-04 16:10:14 -07004134 /* Checked here to keep the fast path fast */
4135 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
4136 if (alloc_flags & ALLOC_NO_WATERMARKS)
4137 goto try_this_zone;
4138
Dave Hansen202e35d2021-05-04 18:36:04 -07004139 if (!node_reclaim_enabled() ||
Mel Gormanc33d6c02016-05-19 17:14:10 -07004140 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
Mel Gormancd38b112011-07-25 17:12:29 -07004141 continue;
4142
Mel Gormana5f5f912016-07-28 15:46:32 -07004143 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
Mel Gormanfa5e0842009-06-16 15:33:22 -07004144 switch (ret) {
Mel Gormana5f5f912016-07-28 15:46:32 -07004145 case NODE_RECLAIM_NOSCAN:
Mel Gormanfa5e0842009-06-16 15:33:22 -07004146 /* did not scan */
Mel Gormancd38b112011-07-25 17:12:29 -07004147 continue;
Mel Gormana5f5f912016-07-28 15:46:32 -07004148 case NODE_RECLAIM_FULL:
Mel Gormanfa5e0842009-06-16 15:33:22 -07004149 /* scanned but unreclaimable */
Mel Gormancd38b112011-07-25 17:12:29 -07004150 continue;
Mel Gormanfa5e0842009-06-16 15:33:22 -07004151 default:
4152 /* did we reclaim enough */
Mel Gormanfed27192013-04-29 15:07:57 -07004153 if (zone_watermark_ok(zone, order, mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004154 ac->highest_zoneidx, alloc_flags))
Mel Gormanfed27192013-04-29 15:07:57 -07004155 goto try_this_zone;
4156
Mel Gormanfed27192013-04-29 15:07:57 -07004157 continue;
Paul Jackson0798e512006-12-06 20:31:38 -08004158 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004159 }
4160
Mel Gormanfa5e0842009-06-16 15:33:22 -07004161try_this_zone:
Mel Gorman066b2392017-02-24 14:56:26 -08004162 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
Mel Gorman0aaa29a2015-11-06 16:28:37 -08004163 gfp_mask, alloc_flags, ac->migratetype);
Vlastimil Babka75379192015-02-11 15:25:38 -08004164 if (page) {
Mel Gorman479f8542016-05-19 17:14:35 -07004165 prep_new_page(page, order, gfp_mask, alloc_flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08004166
4167 /*
4168 * If this is a high-order atomic allocation then check
4169 * if the pageblock should be reserved for the future
4170 */
4171 if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
4172 reserve_highatomic_pageblock(page, zone, order);
4173
Vlastimil Babka75379192015-02-11 15:25:38 -08004174 return page;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07004175 } else {
4176#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
4177 /* Try again if zone has deferred pages */
4178 if (static_branch_unlikely(&deferred_pages)) {
4179 if (_deferred_grow_zone(zone, order))
4180 goto try_this_zone;
4181 }
4182#endif
Vlastimil Babka75379192015-02-11 15:25:38 -08004183 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07004184 }
Paul Jackson9276b1bc2006-12-06 20:31:48 -08004185
Mel Gorman6bb15452018-12-28 00:35:41 -08004186 /*
4187 * It's possible on a UMA machine to get through all zones that are
4188 * fragmented. If avoiding fragmentation, reset and try again.
4189 */
4190 if (no_fallback) {
4191 alloc_flags &= ~ALLOC_NOFRAGMENT;
4192 goto retry;
4193 }
4194
Mel Gorman4ffeaf32014-08-06 16:07:22 -07004195 return NULL;
Martin Hicks753ee722005-06-21 17:14:41 -07004196}
4197
Michal Hocko9af744d2017-02-22 15:46:16 -08004198static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
Dave Hansena238ab52011-05-24 17:12:16 -07004199{
Dave Hansena238ab52011-05-24 17:12:16 -07004200 unsigned int filter = SHOW_MEM_FILTER_NODES;
Dave Hansena238ab52011-05-24 17:12:16 -07004201
4202 /*
4203 * This documents exceptions given to allocations in certain
4204 * contexts that are allowed to allocate outside current's set
4205 * of allowed nodes.
4206 */
4207 if (!(gfp_mask & __GFP_NOMEMALLOC))
Michal Hockocd04ae12017-09-06 16:24:50 -07004208 if (tsk_is_oom_victim(current) ||
Dave Hansena238ab52011-05-24 17:12:16 -07004209 (current->flags & (PF_MEMALLOC | PF_EXITING)))
4210 filter &= ~SHOW_MEM_FILTER_NODES;
Vasily Averin88dc6f202021-09-02 14:58:13 -07004211 if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
Dave Hansena238ab52011-05-24 17:12:16 -07004212 filter &= ~SHOW_MEM_FILTER_NODES;
4213
Michal Hocko9af744d2017-02-22 15:46:16 -08004214 show_mem(filter, nodemask);
Michal Hockoaa187502017-02-22 15:41:45 -08004215}
4216
Michal Hockoa8e99252017-02-22 15:46:10 -08004217void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
Michal Hockoaa187502017-02-22 15:41:45 -08004218{
4219 struct va_format vaf;
4220 va_list args;
Johannes Weiner1be334e2019-11-05 21:16:51 -08004221 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
Michal Hockoaa187502017-02-22 15:41:45 -08004222
Baoquan Hec4dc63f2022-01-14 14:07:44 -08004223 if ((gfp_mask & __GFP_NOWARN) ||
4224 !__ratelimit(&nopage_rs) ||
4225 ((gfp_mask & __GFP_DMA) && !has_managed_dma()))
Michal Hockoaa187502017-02-22 15:41:45 -08004226 return;
4227
Michal Hocko7877cdc2016-10-07 17:01:55 -07004228 va_start(args, fmt);
4229 vaf.fmt = fmt;
4230 vaf.va = &args;
yuzhoujianef8444e2018-12-28 00:36:07 -08004231 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
Michal Hocko0205f752017-11-15 17:39:14 -08004232 current->comm, &vaf, gfp_mask, &gfp_mask,
4233 nodemask_pr_args(nodemask));
Michal Hocko7877cdc2016-10-07 17:01:55 -07004234 va_end(args);
Joe Perches3ee9a4f2011-10-31 17:08:35 -07004235
Michal Hockoa8e99252017-02-22 15:46:10 -08004236 cpuset_print_current_mems_allowed();
yuzhoujianef8444e2018-12-28 00:36:07 -08004237 pr_cont("\n");
Dave Hansena238ab52011-05-24 17:12:16 -07004238 dump_stack();
David Rientjes685dbf62017-02-22 15:46:28 -08004239 warn_alloc_show_mem(gfp_mask, nodemask);
Dave Hansena238ab52011-05-24 17:12:16 -07004240}
4241
Mel Gorman11e33f62009-06-16 15:31:57 -07004242static inline struct page *
Michal Hocko6c18ba72017-02-22 15:46:25 -08004243__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
4244 unsigned int alloc_flags,
4245 const struct alloc_context *ac)
4246{
4247 struct page *page;
4248
4249 page = get_page_from_freelist(gfp_mask, order,
4250 alloc_flags|ALLOC_CPUSET, ac);
4251 /*
4252 * fallback to ignore cpuset restriction if our nodes
4253 * are depleted
4254 */
4255 if (!page)
4256 page = get_page_from_freelist(gfp_mask, order,
4257 alloc_flags, ac);
4258
4259 return page;
4260}
4261
4262static inline struct page *
Mel Gorman11e33f62009-06-16 15:31:57 -07004263__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004264 const struct alloc_context *ac, unsigned long *did_some_progress)
Mel Gorman11e33f62009-06-16 15:31:57 -07004265{
David Rientjes6e0fc462015-09-08 15:00:36 -07004266 struct oom_control oc = {
4267 .zonelist = ac->zonelist,
4268 .nodemask = ac->nodemask,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07004269 .memcg = NULL,
David Rientjes6e0fc462015-09-08 15:00:36 -07004270 .gfp_mask = gfp_mask,
4271 .order = order,
David Rientjes6e0fc462015-09-08 15:00:36 -07004272 };
Linus Torvalds1da177e2005-04-16 15:20:36 -07004273 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004274
Johannes Weiner9879de72015-01-26 12:58:32 -08004275 *did_some_progress = 0;
4276
Johannes Weiner9879de72015-01-26 12:58:32 -08004277 /*
Johannes Weinerdc564012015-06-24 16:57:19 -07004278 * Acquire the oom lock. If that fails, somebody else is
4279 * making progress for us.
Johannes Weiner9879de72015-01-26 12:58:32 -08004280 */
Johannes Weinerdc564012015-06-24 16:57:19 -07004281 if (!mutex_trylock(&oom_lock)) {
Johannes Weiner9879de72015-01-26 12:58:32 -08004282 *did_some_progress = 1;
Mel Gorman11e33f62009-06-16 15:31:57 -07004283 schedule_timeout_uninterruptible(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004284 return NULL;
4285 }
Jens Axboe6b1de912005-11-17 21:35:02 +01004286
Mel Gorman11e33f62009-06-16 15:31:57 -07004287 /*
4288 * Go through the zonelist yet one more time, keep very high watermark
4289 * here, this is only to catch a parallel oom killing, we must fail if
Tetsuo Handae746bf72017-08-31 16:15:20 -07004290 * we're still under heavy pressure. But make sure that this reclaim
4291 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
4292 * allocation which will never fail due to oom_lock already held.
Mel Gorman11e33f62009-06-16 15:31:57 -07004293 */
Tetsuo Handae746bf72017-08-31 16:15:20 -07004294 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
4295 ~__GFP_DIRECT_RECLAIM, order,
4296 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004297 if (page)
Mel Gorman11e33f62009-06-16 15:31:57 -07004298 goto out;
4299
Michal Hocko06ad2762017-02-22 15:46:22 -08004300 /* Coredumps can quickly deplete all memory reserves */
4301 if (current->flags & PF_DUMPCORE)
4302 goto out;
4303 /* The OOM killer will not help higher order allocs */
4304 if (order > PAGE_ALLOC_COSTLY_ORDER)
4305 goto out;
Michal Hockodcda9b02017-07-12 14:36:45 -07004306 /*
4307 * We have already exhausted all our reclaim opportunities without any
4308 * success so it is time to admit defeat. We will skip the OOM killer
4309 * because it is very likely that the caller has a more reasonable
4310 * fallback than shooting a random task.
Mateusz Nosekcfb4a542020-10-13 16:55:45 -07004311 *
4312 * The OOM killer may not free memory on a specific node.
Michal Hockodcda9b02017-07-12 14:36:45 -07004313 */
Mateusz Nosekcfb4a542020-10-13 16:55:45 -07004314 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
Michal Hockodcda9b02017-07-12 14:36:45 -07004315 goto out;
Michal Hocko06ad2762017-02-22 15:46:22 -08004316 /* The OOM killer does not needlessly kill tasks for lowmem */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004317 if (ac->highest_zoneidx < ZONE_NORMAL)
Michal Hocko06ad2762017-02-22 15:46:22 -08004318 goto out;
4319 if (pm_suspended_storage())
4320 goto out;
4321 /*
4322 * XXX: GFP_NOFS allocations should rather fail than rely on
4323 * other request to make a forward progress.
4324 * We are in an unfortunate situation where out_of_memory cannot
4325 * do much for this context but let's try it to at least get
4326 * access to memory reserved if the current task is killed (see
4327 * out_of_memory). Once filesystems are ready to handle allocation
4328 * failures more gracefully we should just bail out here.
4329 */
Michal Hocko3da88fb32016-05-19 17:13:09 -07004330
Shile Zhang3c2c6482018-01-31 16:20:07 -08004331 /* Exhausted what can be done so it's blame time */
Michal Hocko5020e282016-01-14 15:20:36 -08004332 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
Michal Hockoc32b3cb2015-02-11 15:26:24 -08004333 *did_some_progress = 1;
Michal Hocko5020e282016-01-14 15:20:36 -08004334
Michal Hocko6c18ba72017-02-22 15:46:25 -08004335 /*
4336 * Help non-failing allocations by giving them access to memory
4337 * reserves
4338 */
4339 if (gfp_mask & __GFP_NOFAIL)
4340 page = __alloc_pages_cpuset_fallback(gfp_mask, order,
Michal Hocko5020e282016-01-14 15:20:36 -08004341 ALLOC_NO_WATERMARKS, ac);
Michal Hocko5020e282016-01-14 15:20:36 -08004342 }
Mel Gorman11e33f62009-06-16 15:31:57 -07004343out:
Johannes Weinerdc564012015-06-24 16:57:19 -07004344 mutex_unlock(&oom_lock);
Mel Gorman11e33f62009-06-16 15:31:57 -07004345 return page;
4346}
4347
Michal Hocko33c2d212016-05-20 16:57:06 -07004348/*
Lu Jialinbaf2f902021-05-06 18:06:50 -07004349 * Maximum number of compaction retries with a progress before OOM
Michal Hocko33c2d212016-05-20 16:57:06 -07004350 * killer is consider as the only way to move forward.
4351 */
4352#define MAX_COMPACT_RETRIES 16
4353
Mel Gorman56de7262010-05-24 14:32:30 -07004354#ifdef CONFIG_COMPACTION
4355/* Try memory compaction for high-order allocations before reclaim */
4356static struct page *
4357__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004358 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004359 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07004360{
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004361 struct page *page = NULL;
Johannes Weinereb414682018-10-26 15:06:27 -07004362 unsigned long pflags;
Vlastimil Babka499118e2017-05-08 15:59:50 -07004363 unsigned int noreclaim_flag;
Vlastimil Babka53853e22014-10-09 15:27:02 -07004364
Mel Gorman66199712012-01-12 17:19:41 -08004365 if (!order)
Mel Gorman56de7262010-05-24 14:32:30 -07004366 return NULL;
4367
Johannes Weinereb414682018-10-26 15:06:27 -07004368 psi_memstall_enter(&pflags);
wangyong5bf18282022-01-19 18:10:15 -08004369 delayacct_compact_start();
Vlastimil Babka499118e2017-05-08 15:59:50 -07004370 noreclaim_flag = memalloc_noreclaim_save();
Johannes Weinereb414682018-10-26 15:06:27 -07004371
Michal Hockoc5d01d02016-05-20 16:56:53 -07004372 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004373 prio, &page);
Johannes Weinereb414682018-10-26 15:06:27 -07004374
Vlastimil Babka499118e2017-05-08 15:59:50 -07004375 memalloc_noreclaim_restore(noreclaim_flag);
Johannes Weinereb414682018-10-26 15:06:27 -07004376 psi_memstall_leave(&pflags);
wangyong5bf18282022-01-19 18:10:15 -08004377 delayacct_compact_end();
Mel Gorman56de7262010-05-24 14:32:30 -07004378
Charan Teja Reddy06dac2f2021-05-04 18:36:51 -07004379 if (*compact_result == COMPACT_SKIPPED)
4380 return NULL;
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004381 /*
4382 * At least in one zone compaction wasn't deferred or skipped, so let's
4383 * count a compaction stall
4384 */
4385 count_vm_event(COMPACTSTALL);
4386
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004387 /* Prep a captured page if available */
4388 if (page)
4389 prep_new_page(page, order, gfp_mask, alloc_flags);
4390
4391 /* Try get a page from the freelist if available */
4392 if (!page)
4393 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004394
4395 if (page) {
4396 struct zone *zone = page_zone(page);
4397
4398 zone->compact_blockskip_flush = false;
4399 compaction_defer_reset(zone, order, true);
4400 count_vm_event(COMPACTSUCCESS);
4401 return page;
4402 }
4403
4404 /*
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004405 * It's bad if compaction run occurs and fails. The most likely reason
4406 * is that pages exist, but not enough to satisfy watermarks.
4407 */
4408 count_vm_event(COMPACTFAIL);
4409
4410 cond_resched();
4411
Mel Gorman56de7262010-05-24 14:32:30 -07004412 return NULL;
4413}
Michal Hocko33c2d212016-05-20 16:57:06 -07004414
Vlastimil Babka32508452016-10-07 17:00:28 -07004415static inline bool
4416should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
4417 enum compact_result compact_result,
4418 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004419 int *compaction_retries)
Vlastimil Babka32508452016-10-07 17:00:28 -07004420{
4421 int max_retries = MAX_COMPACT_RETRIES;
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004422 int min_priority;
Michal Hocko65190cf2017-02-22 15:42:03 -08004423 bool ret = false;
4424 int retries = *compaction_retries;
4425 enum compact_priority priority = *compact_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07004426
4427 if (!order)
4428 return false;
4429
Aaron Tomlin691d9492021-06-28 19:41:10 -07004430 if (fatal_signal_pending(current))
4431 return false;
4432
Vlastimil Babkad9436492016-10-07 17:00:31 -07004433 if (compaction_made_progress(compact_result))
4434 (*compaction_retries)++;
4435
Vlastimil Babka32508452016-10-07 17:00:28 -07004436 /*
4437 * compaction considers all the zone as desperately out of memory
4438 * so it doesn't really make much sense to retry except when the
4439 * failure could be caused by insufficient priority
4440 */
Vlastimil Babkad9436492016-10-07 17:00:31 -07004441 if (compaction_failed(compact_result))
4442 goto check_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07004443
4444 /*
Vlastimil Babka494330852019-09-23 15:37:32 -07004445 * compaction was skipped because there are not enough order-0 pages
4446 * to work with, so we retry only if it looks like reclaim can help.
Vlastimil Babka32508452016-10-07 17:00:28 -07004447 */
Vlastimil Babka494330852019-09-23 15:37:32 -07004448 if (compaction_needs_reclaim(compact_result)) {
Michal Hocko65190cf2017-02-22 15:42:03 -08004449 ret = compaction_zonelist_suitable(ac, order, alloc_flags);
4450 goto out;
4451 }
Vlastimil Babka32508452016-10-07 17:00:28 -07004452
4453 /*
Vlastimil Babka494330852019-09-23 15:37:32 -07004454 * make sure the compaction wasn't deferred or didn't bail out early
4455 * due to locks contention before we declare that we should give up.
4456 * But the next retry should use a higher priority if allowed, so
4457 * we don't just keep bailing out endlessly.
4458 */
4459 if (compaction_withdrawn(compact_result)) {
4460 goto check_priority;
4461 }
4462
4463 /*
Michal Hockodcda9b02017-07-12 14:36:45 -07004464 * !costly requests are much more important than __GFP_RETRY_MAYFAIL
Vlastimil Babka32508452016-10-07 17:00:28 -07004465 * costly ones because they are de facto nofail and invoke OOM
4466 * killer to move on while costly can fail and users are ready
4467 * to cope with that. 1/4 retries is rather arbitrary but we
4468 * would need much more detailed feedback from compaction to
4469 * make a better decision.
4470 */
4471 if (order > PAGE_ALLOC_COSTLY_ORDER)
4472 max_retries /= 4;
Michal Hocko65190cf2017-02-22 15:42:03 -08004473 if (*compaction_retries <= max_retries) {
4474 ret = true;
4475 goto out;
4476 }
Vlastimil Babka32508452016-10-07 17:00:28 -07004477
Vlastimil Babkad9436492016-10-07 17:00:31 -07004478 /*
4479 * Make sure there are attempts at the highest priority if we exhausted
4480 * all retries or failed at the lower priorities.
4481 */
4482check_priority:
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004483 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
4484 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
Michal Hocko65190cf2017-02-22 15:42:03 -08004485
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004486 if (*compact_priority > min_priority) {
Vlastimil Babkad9436492016-10-07 17:00:31 -07004487 (*compact_priority)--;
4488 *compaction_retries = 0;
Michal Hocko65190cf2017-02-22 15:42:03 -08004489 ret = true;
Vlastimil Babkad9436492016-10-07 17:00:31 -07004490 }
Michal Hocko65190cf2017-02-22 15:42:03 -08004491out:
4492 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
4493 return ret;
Vlastimil Babka32508452016-10-07 17:00:28 -07004494}
Mel Gorman56de7262010-05-24 14:32:30 -07004495#else
4496static inline struct page *
4497__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004498 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004499 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07004500{
Michal Hocko33c2d212016-05-20 16:57:06 -07004501 *compact_result = COMPACT_SKIPPED;
Mel Gorman56de7262010-05-24 14:32:30 -07004502 return NULL;
4503}
Michal Hocko33c2d212016-05-20 16:57:06 -07004504
4505static inline bool
Michal Hocko86a294a2016-05-20 16:57:12 -07004506should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
4507 enum compact_result compact_result,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004508 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004509 int *compaction_retries)
Michal Hocko33c2d212016-05-20 16:57:06 -07004510{
Michal Hocko31e49bf2016-05-20 16:57:15 -07004511 struct zone *zone;
4512 struct zoneref *z;
4513
4514 if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
4515 return false;
4516
4517 /*
4518 * There are setups with compaction disabled which would prefer to loop
4519 * inside the allocator rather than hit the oom killer prematurely.
4520 * Let's give them a good hope and keep retrying while the order-0
4521 * watermarks are OK.
4522 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004523 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4524 ac->highest_zoneidx, ac->nodemask) {
Michal Hocko31e49bf2016-05-20 16:57:15 -07004525 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004526 ac->highest_zoneidx, alloc_flags))
Michal Hocko31e49bf2016-05-20 16:57:15 -07004527 return true;
4528 }
Michal Hocko33c2d212016-05-20 16:57:06 -07004529 return false;
4530}
Vlastimil Babka32508452016-10-07 17:00:28 -07004531#endif /* CONFIG_COMPACTION */
Mel Gorman56de7262010-05-24 14:32:30 -07004532
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004533#ifdef CONFIG_LOCKDEP
Omar Sandoval93781322018-06-07 17:07:02 -07004534static struct lockdep_map __fs_reclaim_map =
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004535 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
4536
Daniel Vetterf920e412020-12-14 19:08:30 -08004537static bool __need_reclaim(gfp_t gfp_mask)
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004538{
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004539 /* no reclaim without waiting on it */
4540 if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
4541 return false;
4542
4543 /* this guy won't enter reclaim */
Tetsuo Handa2e517d682018-03-22 16:17:10 -07004544 if (current->flags & PF_MEMALLOC)
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004545 return false;
4546
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004547 if (gfp_mask & __GFP_NOLOCKDEP)
4548 return false;
4549
4550 return true;
4551}
4552
Matthew Wilcox (Oracle)4f3eaf42021-09-02 14:52:58 -07004553void __fs_reclaim_acquire(unsigned long ip)
Omar Sandoval93781322018-06-07 17:07:02 -07004554{
Matthew Wilcox (Oracle)4f3eaf42021-09-02 14:52:58 -07004555 lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip);
Omar Sandoval93781322018-06-07 17:07:02 -07004556}
4557
Matthew Wilcox (Oracle)4f3eaf42021-09-02 14:52:58 -07004558void __fs_reclaim_release(unsigned long ip)
Omar Sandoval93781322018-06-07 17:07:02 -07004559{
Matthew Wilcox (Oracle)4f3eaf42021-09-02 14:52:58 -07004560 lock_release(&__fs_reclaim_map, ip);
Omar Sandoval93781322018-06-07 17:07:02 -07004561}
4562
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004563void fs_reclaim_acquire(gfp_t gfp_mask)
4564{
Daniel Vetterf920e412020-12-14 19:08:30 -08004565 gfp_mask = current_gfp_context(gfp_mask);
4566
4567 if (__need_reclaim(gfp_mask)) {
4568 if (gfp_mask & __GFP_FS)
Matthew Wilcox (Oracle)4f3eaf42021-09-02 14:52:58 -07004569 __fs_reclaim_acquire(_RET_IP_);
Daniel Vetterf920e412020-12-14 19:08:30 -08004570
4571#ifdef CONFIG_MMU_NOTIFIER
4572 lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
4573 lock_map_release(&__mmu_notifier_invalidate_range_start_map);
4574#endif
4575
4576 }
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004577}
4578EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4579
4580void fs_reclaim_release(gfp_t gfp_mask)
4581{
Daniel Vetterf920e412020-12-14 19:08:30 -08004582 gfp_mask = current_gfp_context(gfp_mask);
4583
4584 if (__need_reclaim(gfp_mask)) {
4585 if (gfp_mask & __GFP_FS)
Matthew Wilcox (Oracle)4f3eaf42021-09-02 14:52:58 -07004586 __fs_reclaim_release(_RET_IP_);
Daniel Vetterf920e412020-12-14 19:08:30 -08004587 }
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004588}
4589EXPORT_SYMBOL_GPL(fs_reclaim_release);
4590#endif
4591
Marek Szyprowskibba90712012-01-25 12:09:52 +01004592/* Perform direct synchronous page reclaim */
Yanfei Xu2187e172020-10-13 16:55:54 -07004593static unsigned long
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004594__perform_reclaim(gfp_t gfp_mask, unsigned int order,
4595 const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004596{
Vlastimil Babka499118e2017-05-08 15:59:50 -07004597 unsigned int noreclaim_flag;
Yanfei Xu2187e172020-10-13 16:55:54 -07004598 unsigned long pflags, progress;
Mel Gorman11e33f62009-06-16 15:31:57 -07004599
4600 cond_resched();
4601
4602 /* We now go into synchronous reclaim */
4603 cpuset_memory_pressure_bump();
Johannes Weinereb414682018-10-26 15:06:27 -07004604 psi_memstall_enter(&pflags);
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004605 fs_reclaim_acquire(gfp_mask);
Omar Sandoval93781322018-06-07 17:07:02 -07004606 noreclaim_flag = memalloc_noreclaim_save();
Mel Gorman11e33f62009-06-16 15:31:57 -07004607
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004608 progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4609 ac->nodemask);
Mel Gorman11e33f62009-06-16 15:31:57 -07004610
Vlastimil Babka499118e2017-05-08 15:59:50 -07004611 memalloc_noreclaim_restore(noreclaim_flag);
Omar Sandoval93781322018-06-07 17:07:02 -07004612 fs_reclaim_release(gfp_mask);
Johannes Weinereb414682018-10-26 15:06:27 -07004613 psi_memstall_leave(&pflags);
Mel Gorman11e33f62009-06-16 15:31:57 -07004614
4615 cond_resched();
4616
Marek Szyprowskibba90712012-01-25 12:09:52 +01004617 return progress;
4618}
4619
4620/* The really slow allocator path where we enter direct reclaim */
4621static inline struct page *
4622__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004623 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004624 unsigned long *did_some_progress)
Marek Szyprowskibba90712012-01-25 12:09:52 +01004625{
4626 struct page *page = NULL;
4627 bool drained = false;
4628
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004629 *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004630 if (unlikely(!(*did_some_progress)))
4631 return NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07004632
Mel Gorman9ee493c2010-09-09 16:38:18 -07004633retry:
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004634 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004635
4636 /*
4637 * If an allocation failed after direct reclaim, it could be because
Mel Gorman0aaa29a2015-11-06 16:28:37 -08004638 * pages are pinned on the per-cpu lists or in high alloc reserves.
Randy Dunlap047b9962020-08-11 18:33:14 -07004639 * Shrink them and try again
Mel Gorman9ee493c2010-09-09 16:38:18 -07004640 */
4641 if (!page && !drained) {
Minchan Kim29fac032016-12-12 16:42:14 -08004642 unreserve_highatomic_pageblock(ac, false);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08004643 drain_all_pages(NULL);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004644 drained = true;
4645 goto retry;
4646 }
4647
Mel Gorman11e33f62009-06-16 15:31:57 -07004648 return page;
4649}
4650
David Rientjes5ecd9d42018-04-05 16:25:16 -07004651static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4652 const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004653{
4654 struct zoneref *z;
4655 struct zone *zone;
Mel Gormane1a55632016-07-28 15:46:26 -07004656 pg_data_t *last_pgdat = NULL;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004657 enum zone_type highest_zoneidx = ac->highest_zoneidx;
Mel Gorman11e33f62009-06-16 15:31:57 -07004658
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004659 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
David Rientjes5ecd9d42018-04-05 16:25:16 -07004660 ac->nodemask) {
Mel Gormane1a55632016-07-28 15:46:26 -07004661 if (last_pgdat != zone->zone_pgdat)
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004662 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);
Mel Gormane1a55632016-07-28 15:46:26 -07004663 last_pgdat = zone->zone_pgdat;
4664 }
Mel Gorman11e33f62009-06-16 15:31:57 -07004665}
4666
Mel Gormanc6038442016-05-19 17:13:38 -07004667static inline unsigned int
Peter Zijlstra341ce062009-06-16 15:32:02 -07004668gfp_to_alloc_flags(gfp_t gfp_mask)
4669{
Mel Gormanc6038442016-05-19 17:13:38 -07004670 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
Peter Zijlstra341ce062009-06-16 15:32:02 -07004671
Mateusz Nosek736838e2020-04-01 21:09:47 -07004672 /*
4673 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH
4674 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4675 * to save two branches.
4676 */
Namhyung Kime6223a32010-10-26 14:21:59 -07004677 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
Mateusz Nosek736838e2020-04-01 21:09:47 -07004678 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
Mel Gormana56f57f2009-06-16 15:32:02 -07004679
Peter Zijlstra341ce062009-06-16 15:32:02 -07004680 /*
4681 * The caller may dip into page reserves a bit more if the caller
4682 * cannot run direct reclaim, or if the caller has realtime scheduling
4683 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
Mel Gormand0164ad2015-11-06 16:28:21 -08004684 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
Peter Zijlstra341ce062009-06-16 15:32:02 -07004685 */
Mateusz Nosek736838e2020-04-01 21:09:47 -07004686 alloc_flags |= (__force int)
4687 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
Peter Zijlstra341ce062009-06-16 15:32:02 -07004688
Mel Gormand0164ad2015-11-06 16:28:21 -08004689 if (gfp_mask & __GFP_ATOMIC) {
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004690 /*
David Rientjesb104a352014-07-30 16:08:24 -07004691 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
4692 * if it can't schedule.
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004693 */
David Rientjesb104a352014-07-30 16:08:24 -07004694 if (!(gfp_mask & __GFP_NOMEMALLOC))
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004695 alloc_flags |= ALLOC_HARDER;
Peter Zijlstra341ce062009-06-16 15:32:02 -07004696 /*
David Rientjesb104a352014-07-30 16:08:24 -07004697 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
Vladimir Davydov344736f2014-10-20 15:50:30 +04004698 * comment for __cpuset_node_allowed().
Peter Zijlstra341ce062009-06-16 15:32:02 -07004699 */
4700 alloc_flags &= ~ALLOC_CPUSET;
Vasily Averin88dc6f202021-09-02 14:58:13 -07004701 } else if (unlikely(rt_task(current)) && in_task())
Peter Zijlstra341ce062009-06-16 15:32:02 -07004702 alloc_flags |= ALLOC_HARDER;
4703
Pavel Tatashin8e3560d2021-05-04 18:39:00 -07004704 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);
Joonsoo Kim8510e692020-08-06 23:26:04 -07004705
Peter Zijlstra341ce062009-06-16 15:32:02 -07004706 return alloc_flags;
4707}
4708
Michal Hockocd04ae12017-09-06 16:24:50 -07004709static bool oom_reserves_allowed(struct task_struct *tsk)
Mel Gorman072bb0a2012-07-31 16:43:58 -07004710{
Michal Hockocd04ae12017-09-06 16:24:50 -07004711 if (!tsk_is_oom_victim(tsk))
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004712 return false;
4713
Michal Hockocd04ae12017-09-06 16:24:50 -07004714 /*
4715 * !MMU doesn't have oom reaper so give access to memory reserves
4716 * only to the thread with TIF_MEMDIE set
4717 */
4718 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4719 return false;
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004720
Michal Hockocd04ae12017-09-06 16:24:50 -07004721 return true;
4722}
4723
4724/*
4725 * Distinguish requests which really need access to full memory
4726 * reserves from oom victims which can live with a portion of it
4727 */
4728static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4729{
4730 if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4731 return 0;
4732 if (gfp_mask & __GFP_MEMALLOC)
4733 return ALLOC_NO_WATERMARKS;
4734 if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4735 return ALLOC_NO_WATERMARKS;
4736 if (!in_interrupt()) {
4737 if (current->flags & PF_MEMALLOC)
4738 return ALLOC_NO_WATERMARKS;
4739 else if (oom_reserves_allowed(current))
4740 return ALLOC_OOM;
4741 }
4742
4743 return 0;
4744}
4745
4746bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4747{
4748 return !!__gfp_pfmemalloc_flags(gfp_mask);
Mel Gorman072bb0a2012-07-31 16:43:58 -07004749}
4750
Michal Hocko0a0337e2016-05-20 16:57:00 -07004751/*
Michal Hocko0a0337e2016-05-20 16:57:00 -07004752 * Checks whether it makes sense to retry the reclaim to make a forward progress
4753 * for the given allocation request.
Johannes Weiner491d79a2017-05-03 14:52:16 -07004754 *
4755 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
4756 * without success, or when we couldn't even meet the watermark if we
4757 * reclaimed all remaining pages on the LRU lists.
Michal Hocko0a0337e2016-05-20 16:57:00 -07004758 *
4759 * Returns true if a retry is viable or false to enter the oom path.
4760 */
4761static inline bool
4762should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4763 struct alloc_context *ac, int alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07004764 bool did_some_progress, int *no_progress_loops)
Michal Hocko0a0337e2016-05-20 16:57:00 -07004765{
4766 struct zone *zone;
4767 struct zoneref *z;
Michal Hocko15f570b2018-10-26 15:03:31 -07004768 bool ret = false;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004769
4770 /*
Vlastimil Babka423b4522016-10-07 17:00:40 -07004771 * Costly allocations might have made a progress but this doesn't mean
4772 * their order will become available due to high fragmentation so
4773 * always increment the no progress counter for them
4774 */
4775 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4776 *no_progress_loops = 0;
4777 else
4778 (*no_progress_loops)++;
4779
4780 /*
Michal Hocko0a0337e2016-05-20 16:57:00 -07004781 * Make sure we converge to OOM if we cannot make any progress
4782 * several times in the row.
4783 */
Minchan Kim04c87162016-12-12 16:42:11 -08004784 if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4785 /* Before OOM, exhaust highatomic_reserve */
Minchan Kim29fac032016-12-12 16:42:14 -08004786 return unreserve_highatomic_pageblock(ac, true);
Minchan Kim04c87162016-12-12 16:42:11 -08004787 }
Michal Hocko0a0337e2016-05-20 16:57:00 -07004788
Michal Hocko0a0337e2016-05-20 16:57:00 -07004789 /*
Mel Gormanbca67592016-07-28 15:47:05 -07004790 * Keep reclaiming pages while there is a chance this will lead
4791 * somewhere. If none of the target zones can satisfy our allocation
4792 * request even if all reclaimable pages are considered then we are
4793 * screwed and have to go OOM.
Michal Hocko0a0337e2016-05-20 16:57:00 -07004794 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004795 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4796 ac->highest_zoneidx, ac->nodemask) {
Michal Hocko0a0337e2016-05-20 16:57:00 -07004797 unsigned long available;
Michal Hockoede37712016-05-20 16:57:03 -07004798 unsigned long reclaimable;
Michal Hockod379f012017-02-22 15:42:00 -08004799 unsigned long min_wmark = min_wmark_pages(zone);
4800 bool wmark;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004801
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004802 available = reclaimable = zone_reclaimable_pages(zone);
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004803 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
Michal Hocko0a0337e2016-05-20 16:57:00 -07004804
4805 /*
Johannes Weiner491d79a2017-05-03 14:52:16 -07004806 * Would the allocation succeed if we reclaimed all
4807 * reclaimable pages?
Michal Hocko0a0337e2016-05-20 16:57:00 -07004808 */
Michal Hockod379f012017-02-22 15:42:00 -08004809 wmark = __zone_watermark_ok(zone, order, min_wmark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004810 ac->highest_zoneidx, alloc_flags, available);
Michal Hockod379f012017-02-22 15:42:00 -08004811 trace_reclaim_retry_zone(z, order, reclaimable,
4812 available, min_wmark, *no_progress_loops, wmark);
4813 if (wmark) {
Michal Hocko15f570b2018-10-26 15:03:31 -07004814 ret = true;
Mel Gorman132b0d22021-11-05 13:42:38 -07004815 break;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004816 }
4817 }
4818
Michal Hocko15f570b2018-10-26 15:03:31 -07004819 /*
4820 * Memory allocation/reclaim might be called from a WQ context and the
4821 * current implementation of the WQ concurrency control doesn't
4822 * recognize that a particular WQ is congested if the worker thread is
4823 * looping without ever sleeping. Therefore we have to do a short sleep
4824 * here rather than calling cond_resched().
4825 */
4826 if (current->flags & PF_WQ_WORKER)
4827 schedule_timeout_uninterruptible(1);
4828 else
4829 cond_resched();
4830 return ret;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004831}
4832
Vlastimil Babka902b6282017-07-06 15:39:56 -07004833static inline bool
4834check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4835{
4836 /*
4837 * It's possible that cpuset's mems_allowed and the nodemask from
4838 * mempolicy don't intersect. This should be normally dealt with by
4839 * policy_nodemask(), but it's possible to race with cpuset update in
4840 * such a way the check therein was true, and then it became false
4841 * before we got our cpuset_mems_cookie here.
4842 * This assumes that for all allocations, ac->nodemask can come only
4843 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
4844 * when it does not intersect with the cpuset restrictions) or the
4845 * caller can deal with a violated nodemask.
4846 */
4847 if (cpusets_enabled() && ac->nodemask &&
4848 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4849 ac->nodemask = NULL;
4850 return true;
4851 }
4852
4853 /*
4854 * When updating a task's mems_allowed or mempolicy nodemask, it is
4855 * possible to race with parallel threads in such a way that our
4856 * allocation can fail while the mask is being updated. If we are about
4857 * to fail, check if the cpuset changed during allocation and if so,
4858 * retry.
4859 */
4860 if (read_mems_allowed_retry(cpuset_mems_cookie))
4861 return true;
4862
4863 return false;
4864}
4865
Mel Gorman11e33f62009-06-16 15:31:57 -07004866static inline struct page *
4867__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004868 struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004869{
Mel Gormand0164ad2015-11-06 16:28:21 -08004870 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
Vlastimil Babka282722b2017-05-08 15:54:49 -07004871 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
Mel Gorman11e33f62009-06-16 15:31:57 -07004872 struct page *page = NULL;
Mel Gormanc6038442016-05-19 17:13:38 -07004873 unsigned int alloc_flags;
Mel Gorman11e33f62009-06-16 15:31:57 -07004874 unsigned long did_some_progress;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004875 enum compact_priority compact_priority;
Michal Hockoc5d01d02016-05-20 16:56:53 -07004876 enum compact_result compact_result;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004877 int compaction_retries;
4878 int no_progress_loops;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004879 unsigned int cpuset_mems_cookie;
Michal Hockocd04ae12017-09-06 16:24:50 -07004880 int reserve_flags;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004881
Christoph Lameter952f3b52006-12-06 20:33:26 -08004882 /*
Mel Gormand0164ad2015-11-06 16:28:21 -08004883 * We also sanity check to catch abuse of atomic reserves being used by
4884 * callers that are not in atomic context.
4885 */
4886 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4887 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4888 gfp_mask &= ~__GFP_ATOMIC;
4889
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004890retry_cpuset:
4891 compaction_retries = 0;
4892 no_progress_loops = 0;
4893 compact_priority = DEF_COMPACT_PRIORITY;
4894 cpuset_mems_cookie = read_mems_allowed_begin();
Michal Hocko9a67f642017-02-22 15:46:19 -08004895
4896 /*
4897 * The fast path uses conservative alloc_flags to succeed only until
4898 * kswapd needs to be woken up, and to avoid the cost of setting up
4899 * alloc_flags precisely. So we do that now.
4900 */
4901 alloc_flags = gfp_to_alloc_flags(gfp_mask);
4902
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004903 /*
4904 * We need to recalculate the starting point for the zonelist iterator
4905 * because we might have used different nodemask in the fast path, or
4906 * there was a cpuset modification and we are retrying - otherwise we
4907 * could end up iterating over non-eligible zones endlessly.
4908 */
4909 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004910 ac->highest_zoneidx, ac->nodemask);
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004911 if (!ac->preferred_zoneref->zone)
4912 goto nopage;
4913
Feng Tang8ca1b5a2021-11-05 13:40:34 -07004914 /*
4915 * Check for insane configurations where the cpuset doesn't contain
4916 * any suitable zone to satisfy the request - e.g. non-movable
4917 * GFP_HIGHUSER allocations from MOVABLE nodes only.
4918 */
4919 if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) {
4920 struct zoneref *z = first_zones_zonelist(ac->zonelist,
4921 ac->highest_zoneidx,
4922 &cpuset_current_mems_allowed);
4923 if (!z->zone)
4924 goto nopage;
4925 }
4926
Mel Gorman0a79cda2018-12-28 00:35:48 -08004927 if (alloc_flags & ALLOC_KSWAPD)
David Rientjes5ecd9d42018-04-05 16:25:16 -07004928 wake_all_kswapds(order, gfp_mask, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004929
Paul Jackson9bf22292005-09-06 15:18:12 -07004930 /*
Vlastimil Babka23771232016-07-28 15:49:16 -07004931 * The adjusted alloc_flags might result in immediate success, so try
4932 * that first
4933 */
4934 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4935 if (page)
4936 goto got_pg;
4937
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004938 /*
4939 * For costly allocations, try direct compaction first, as it's likely
Vlastimil Babka282722b2017-05-08 15:54:49 -07004940 * that we have enough base pages and don't need to reclaim. For non-
4941 * movable high-order allocations, do that as well, as compaction will
4942 * try prevent permanent fragmentation by migrating from blocks of the
4943 * same migratetype.
4944 * Don't try this for allocations that are allowed to ignore
4945 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004946 */
Vlastimil Babka282722b2017-05-08 15:54:49 -07004947 if (can_direct_reclaim &&
4948 (costly_order ||
4949 (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4950 && !gfp_pfmemalloc_allowed(gfp_mask)) {
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004951 page = __alloc_pages_direct_compact(gfp_mask, order,
4952 alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004953 INIT_COMPACT_PRIORITY,
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004954 &compact_result);
4955 if (page)
4956 goto got_pg;
4957
Vlastimil Babkacc638f32020-01-13 16:29:04 -08004958 /*
4959 * Checks for costly allocations with __GFP_NORETRY, which
4960 * includes some THP page fault allocations
4961 */
4962 if (costly_order && (gfp_mask & __GFP_NORETRY)) {
David Rientjesb39d0ee2019-09-04 12:54:22 -07004963 /*
4964 * If allocating entire pageblock(s) and compaction
4965 * failed because all zones are below low watermarks
4966 * or is prohibited because it recently failed at this
David Rientjes3f36d862019-10-14 14:12:04 -07004967 * order, fail immediately unless the allocator has
4968 * requested compaction and reclaim retry.
David Rientjesb39d0ee2019-09-04 12:54:22 -07004969 *
4970 * Reclaim is
4971 * - potentially very expensive because zones are far
4972 * below their low watermarks or this is part of very
4973 * bursty high order allocations,
4974 * - not guaranteed to help because isolate_freepages()
4975 * may not iterate over freed pages as part of its
4976 * linear scan, and
4977 * - unlikely to make entire pageblocks free on its
4978 * own.
4979 */
4980 if (compact_result == COMPACT_SKIPPED ||
4981 compact_result == COMPACT_DEFERRED)
4982 goto nopage;
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004983
4984 /*
Vlastimil Babka3eb27712016-07-28 15:49:22 -07004985 * Looks like reclaim/compaction is worth trying, but
4986 * sync compaction could be very expensive, so keep
Vlastimil Babka25160352016-07-28 15:49:25 -07004987 * using async compaction.
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004988 */
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004989 compact_priority = INIT_COMPACT_PRIORITY;
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004990 }
4991 }
Vlastimil Babka23771232016-07-28 15:49:16 -07004992
4993retry:
4994 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
Mel Gorman0a79cda2018-12-28 00:35:48 -08004995 if (alloc_flags & ALLOC_KSWAPD)
David Rientjes5ecd9d42018-04-05 16:25:16 -07004996 wake_all_kswapds(order, gfp_mask, ac);
Vlastimil Babka23771232016-07-28 15:49:16 -07004997
Michal Hockocd04ae12017-09-06 16:24:50 -07004998 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4999 if (reserve_flags)
Pavel Tatashin8e3560d2021-05-04 18:39:00 -07005000 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags);
Vlastimil Babka23771232016-07-28 15:49:16 -07005001
5002 /*
Vlastimil Babkad6a24df2018-08-17 15:45:05 -07005003 * Reset the nodemask and zonelist iterators if memory policies can be
5004 * ignored. These allocations are high priority and system rather than
5005 * user oriented.
Mel Gormane46e7b72016-06-03 14:56:01 -07005006 */
Michal Hockocd04ae12017-09-06 16:24:50 -07005007 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
Vlastimil Babkad6a24df2018-08-17 15:45:05 -07005008 ac->nodemask = NULL;
Mel Gormane46e7b72016-06-03 14:56:01 -07005009 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07005010 ac->highest_zoneidx, ac->nodemask);
Mel Gormane46e7b72016-06-03 14:56:01 -07005011 }
5012
Vlastimil Babka23771232016-07-28 15:49:16 -07005013 /* Attempt with potentially adjusted zonelist and alloc_flags */
Vlastimil Babka31a6c192016-07-28 15:49:13 -07005014 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08005015 if (page)
5016 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005017
Mel Gormand0164ad2015-11-06 16:28:21 -08005018 /* Caller is not willing to reclaim, we can't balance anything */
Michal Hocko9a67f642017-02-22 15:46:19 -08005019 if (!can_direct_reclaim)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005020 goto nopage;
Michal Hocko9a67f642017-02-22 15:46:19 -08005021
Peter Zijlstra341ce062009-06-16 15:32:02 -07005022 /* Avoid recursion of direct reclaim */
Michal Hocko9a67f642017-02-22 15:46:19 -08005023 if (current->flags & PF_MEMALLOC)
Peter Zijlstra341ce062009-06-16 15:32:02 -07005024 goto nopage;
David Rientjes8fe78042014-08-06 16:07:54 -07005025
Mel Gorman11e33f62009-06-16 15:31:57 -07005026 /* Try direct reclaim and then allocating */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08005027 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
5028 &did_some_progress);
Mel Gorman11e33f62009-06-16 15:31:57 -07005029 if (page)
5030 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005031
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07005032 /* Try direct compaction and then allocating */
5033 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07005034 compact_priority, &compact_result);
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07005035 if (page)
5036 goto got_pg;
5037
Johannes Weiner90839052015-06-24 16:57:21 -07005038 /* Do not loop if specifically requested */
5039 if (gfp_mask & __GFP_NORETRY)
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07005040 goto nopage;
Johannes Weiner90839052015-06-24 16:57:21 -07005041
Michal Hocko0a0337e2016-05-20 16:57:00 -07005042 /*
5043 * Do not retry costly high order allocations unless they are
Michal Hockodcda9b02017-07-12 14:36:45 -07005044 * __GFP_RETRY_MAYFAIL
Michal Hocko0a0337e2016-05-20 16:57:00 -07005045 */
Michal Hockodcda9b02017-07-12 14:36:45 -07005046 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07005047 goto nopage;
Michal Hocko0a0337e2016-05-20 16:57:00 -07005048
Michal Hocko0a0337e2016-05-20 16:57:00 -07005049 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07005050 did_some_progress > 0, &no_progress_loops))
Michal Hocko0a0337e2016-05-20 16:57:00 -07005051 goto retry;
5052
Michal Hocko33c2d212016-05-20 16:57:06 -07005053 /*
5054 * It doesn't make any sense to retry for the compaction if the order-0
5055 * reclaim is not able to make any progress because the current
5056 * implementation of the compaction depends on the sufficient amount
5057 * of free memory (see __compaction_suitable)
5058 */
5059 if (did_some_progress > 0 &&
Michal Hocko86a294a2016-05-20 16:57:12 -07005060 should_compact_retry(ac, order, alloc_flags,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07005061 compact_result, &compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07005062 &compaction_retries))
Michal Hocko33c2d212016-05-20 16:57:06 -07005063 goto retry;
5064
Vlastimil Babka902b6282017-07-06 15:39:56 -07005065
5066 /* Deal with possible cpuset update races before we start OOM killing */
5067 if (check_retry_cpuset(cpuset_mems_cookie, ac))
Vlastimil Babkae47483b2017-01-24 15:18:41 -08005068 goto retry_cpuset;
5069
Johannes Weiner90839052015-06-24 16:57:21 -07005070 /* Reclaim has failed us, start killing things */
5071 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
5072 if (page)
5073 goto got_pg;
5074
Michal Hocko9a67f642017-02-22 15:46:19 -08005075 /* Avoid allocations with no watermarks from looping endlessly */
Michal Hockocd04ae12017-09-06 16:24:50 -07005076 if (tsk_is_oom_victim(current) &&
Joonsoo Kim8510e692020-08-06 23:26:04 -07005077 (alloc_flags & ALLOC_OOM ||
Tetsuo Handac2889832017-06-02 14:46:31 -07005078 (gfp_mask & __GFP_NOMEMALLOC)))
Michal Hocko9a67f642017-02-22 15:46:19 -08005079 goto nopage;
5080
Johannes Weiner90839052015-06-24 16:57:21 -07005081 /* Retry as long as the OOM killer is making progress */
Michal Hocko0a0337e2016-05-20 16:57:00 -07005082 if (did_some_progress) {
5083 no_progress_loops = 0;
Johannes Weiner90839052015-06-24 16:57:21 -07005084 goto retry;
Michal Hocko0a0337e2016-05-20 16:57:00 -07005085 }
Johannes Weiner90839052015-06-24 16:57:21 -07005086
Linus Torvalds1da177e2005-04-16 15:20:36 -07005087nopage:
Vlastimil Babka902b6282017-07-06 15:39:56 -07005088 /* Deal with possible cpuset update races before we fail */
5089 if (check_retry_cpuset(cpuset_mems_cookie, ac))
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08005090 goto retry_cpuset;
5091
Michal Hocko9a67f642017-02-22 15:46:19 -08005092 /*
5093 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
5094 * we always retry
5095 */
5096 if (gfp_mask & __GFP_NOFAIL) {
5097 /*
5098 * All existing users of the __GFP_NOFAIL are blockable, so warn
5099 * of any new users that actually require GFP_NOWAIT
5100 */
5101 if (WARN_ON_ONCE(!can_direct_reclaim))
5102 goto fail;
5103
5104 /*
5105 * PF_MEMALLOC request from this context is rather bizarre
5106 * because we cannot reclaim anything and only can loop waiting
5107 * for somebody to do a work for us
5108 */
5109 WARN_ON_ONCE(current->flags & PF_MEMALLOC);
5110
5111 /*
5112 * non failing costly orders are a hard requirement which we
5113 * are not prepared for much so let's warn about these users
5114 * so that we can identify them and convert them to something
5115 * else.
5116 */
5117 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
5118
Michal Hocko6c18ba72017-02-22 15:46:25 -08005119 /*
5120 * Help non-failing allocations by giving them access to memory
5121 * reserves but do not use ALLOC_NO_WATERMARKS because this
5122 * could deplete whole memory reserves which would just make
5123 * the situation worse
5124 */
5125 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
5126 if (page)
5127 goto got_pg;
5128
Michal Hocko9a67f642017-02-22 15:46:19 -08005129 cond_resched();
5130 goto retry;
5131 }
5132fail:
Michal Hockoa8e99252017-02-22 15:46:10 -08005133 warn_alloc(gfp_mask, ac->nodemask,
Michal Hocko7877cdc2016-10-07 17:01:55 -07005134 "page allocation failure: order:%u", order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005135got_pg:
Mel Gorman072bb0a2012-07-31 16:43:58 -07005136 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005137}
Mel Gorman11e33f62009-06-16 15:31:57 -07005138
Mel Gorman9cd75552017-02-24 14:56:29 -08005139static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
Vlastimil Babka04ec6262017-07-06 15:40:03 -07005140 int preferred_nid, nodemask_t *nodemask,
Matthew Wilcox (Oracle)8e6a9302021-04-29 23:01:10 -07005141 struct alloc_context *ac, gfp_t *alloc_gfp,
Mel Gorman9cd75552017-02-24 14:56:29 -08005142 unsigned int *alloc_flags)
5143{
Joonsoo Kim97a225e2020-06-03 15:59:01 -07005144 ac->highest_zoneidx = gfp_zone(gfp_mask);
Vlastimil Babka04ec6262017-07-06 15:40:03 -07005145 ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08005146 ac->nodemask = nodemask;
Wei Yang01c0bfe2020-06-03 15:59:08 -07005147 ac->migratetype = gfp_migratetype(gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08005148
5149 if (cpusets_enabled()) {
Matthew Wilcox (Oracle)8e6a9302021-04-29 23:01:10 -07005150 *alloc_gfp |= __GFP_HARDWALL;
Muchun Song182f3d72020-08-06 23:26:01 -07005151 /*
5152 * When we are in the interrupt context, it is irrelevant
5153 * to the current task context. It means that any node ok.
5154 */
Vasily Averin88dc6f202021-09-02 14:58:13 -07005155 if (in_task() && !ac->nodemask)
Mel Gorman9cd75552017-02-24 14:56:29 -08005156 ac->nodemask = &cpuset_current_mems_allowed;
Vlastimil Babka51047822017-02-24 14:56:53 -08005157 else
5158 *alloc_flags |= ALLOC_CPUSET;
Mel Gorman9cd75552017-02-24 14:56:29 -08005159 }
5160
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01005161 fs_reclaim_acquire(gfp_mask);
5162 fs_reclaim_release(gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08005163
5164 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
5165
5166 if (should_fail_alloc_page(gfp_mask, order))
5167 return false;
5168
Pavel Tatashin8e3560d2021-05-04 18:39:00 -07005169 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09005170
Mel Gorman9cd75552017-02-24 14:56:29 -08005171 /* Dirty zone balancing only done in the fast path */
5172 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
5173
5174 /*
5175 * The preferred zone is used for statistics but crucially it is
5176 * also used as the starting point for the zonelist iterator. It
5177 * may get reset for allocations that ignore memory policies.
5178 */
5179 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07005180 ac->highest_zoneidx, ac->nodemask);
Mateusz Noseka0622d02020-10-13 16:55:51 -07005181
5182 return true;
Mel Gorman9cd75552017-02-24 14:56:29 -08005183}
5184
Mel Gorman11e33f62009-06-16 15:31:57 -07005185/*
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005186 * __alloc_pages_bulk - Allocate a number of order-0 pages to a list or array
Mel Gorman387ba26f2021-04-29 23:01:45 -07005187 * @gfp: GFP flags for the allocation
5188 * @preferred_nid: The preferred NUMA node ID to allocate from
5189 * @nodemask: Set of nodes to allocate from, may be NULL
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005190 * @nr_pages: The number of pages desired on the list or array
5191 * @page_list: Optional list to store the allocated pages
5192 * @page_array: Optional array to store the pages
Mel Gorman387ba26f2021-04-29 23:01:45 -07005193 *
5194 * This is a batched version of the page allocator that attempts to
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005195 * allocate nr_pages quickly. Pages are added to page_list if page_list
5196 * is not NULL, otherwise it is assumed that the page_array is valid.
Mel Gorman387ba26f2021-04-29 23:01:45 -07005197 *
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005198 * For lists, nr_pages is the number of pages that should be allocated.
5199 *
5200 * For arrays, only NULL elements are populated with pages and nr_pages
5201 * is the maximum number of pages that will be stored in the array.
5202 *
5203 * Returns the number of pages on the list or array.
Mel Gorman387ba26f2021-04-29 23:01:45 -07005204 */
5205unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
5206 nodemask_t *nodemask, int nr_pages,
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005207 struct list_head *page_list,
5208 struct page **page_array)
Mel Gorman387ba26f2021-04-29 23:01:45 -07005209{
5210 struct page *page;
5211 unsigned long flags;
5212 struct zone *zone;
5213 struct zoneref *z;
5214 struct per_cpu_pages *pcp;
5215 struct list_head *pcp_list;
5216 struct alloc_context ac;
5217 gfp_t alloc_gfp;
5218 unsigned int alloc_flags = ALLOC_WMARK_LOW;
Mel Gorman3e230602021-06-28 19:41:50 -07005219 int nr_populated = 0, nr_account = 0;
Mel Gorman387ba26f2021-04-29 23:01:45 -07005220
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005221 /*
5222 * Skip populated array elements to determine if any pages need
5223 * to be allocated before disabling IRQs.
5224 */
Rasmus Villemoesb08e50d2021-06-24 18:40:04 -07005225 while (page_array && nr_populated < nr_pages && page_array[nr_populated])
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005226 nr_populated++;
5227
Chuck Lever06147842021-07-14 21:26:52 -07005228 /* No pages requested? */
5229 if (unlikely(nr_pages <= 0))
5230 goto out;
5231
Mel Gormanb3b64eb2021-06-24 18:40:07 -07005232 /* Already populated array? */
5233 if (unlikely(page_array && nr_pages - nr_populated == 0))
Chuck Lever06147842021-07-14 21:26:52 -07005234 goto out;
Mel Gormanb3b64eb2021-06-24 18:40:07 -07005235
Shakeel Butt8dcb3062021-10-28 14:36:04 -07005236 /* Bulk allocator does not support memcg accounting. */
5237 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT))
5238 goto failed;
5239
Mel Gorman387ba26f2021-04-29 23:01:45 -07005240 /* Use the single page allocator for one page. */
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005241 if (nr_pages - nr_populated == 1)
Mel Gorman387ba26f2021-04-29 23:01:45 -07005242 goto failed;
5243
Mel Gorman187ad462021-07-14 21:26:46 -07005244#ifdef CONFIG_PAGE_OWNER
5245 /*
5246 * PAGE_OWNER may recurse into the allocator to allocate space to
5247 * save the stack with pagesets.lock held. Releasing/reacquiring
5248 * removes much of the performance benefit of bulk allocation so
5249 * force the caller to allocate one page at a time as it'll have
5250 * similar performance to added complexity to the bulk allocator.
5251 */
5252 if (static_branch_unlikely(&page_owner_inited))
5253 goto failed;
5254#endif
5255
Mel Gorman387ba26f2021-04-29 23:01:45 -07005256 /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
5257 gfp &= gfp_allowed_mask;
5258 alloc_gfp = gfp;
5259 if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags))
Chuck Lever06147842021-07-14 21:26:52 -07005260 goto out;
Mel Gorman387ba26f2021-04-29 23:01:45 -07005261 gfp = alloc_gfp;
5262
5263 /* Find an allowed local zone that meets the low watermark. */
5264 for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) {
5265 unsigned long mark;
5266
5267 if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) &&
5268 !__cpuset_zone_allowed(zone, gfp)) {
5269 continue;
5270 }
5271
5272 if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone &&
5273 zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) {
5274 goto failed;
5275 }
5276
5277 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages;
5278 if (zone_watermark_fast(zone, 0, mark,
5279 zonelist_zone_idx(ac.preferred_zoneref),
5280 alloc_flags, gfp)) {
5281 break;
5282 }
5283 }
5284
5285 /*
5286 * If there are no allowed local zones that meets the watermarks then
5287 * try to allocate a single page and reclaim if necessary.
5288 */
Jesper Dangaard Brouerce76f9a2021-04-29 23:01:51 -07005289 if (unlikely(!zone))
Mel Gorman387ba26f2021-04-29 23:01:45 -07005290 goto failed;
5291
5292 /* Attempt the batch allocation */
Mel Gormandbbee9d2021-06-28 19:41:41 -07005293 local_lock_irqsave(&pagesets.lock, flags);
Mel Gorman28f836b2021-06-28 19:41:38 -07005294 pcp = this_cpu_ptr(zone->per_cpu_pageset);
Mel Gorman44042b42021-06-28 19:43:08 -07005295 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)];
Mel Gorman387ba26f2021-04-29 23:01:45 -07005296
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005297 while (nr_populated < nr_pages) {
5298
5299 /* Skip existing pages */
5300 if (page_array && page_array[nr_populated]) {
5301 nr_populated++;
5302 continue;
5303 }
5304
Mel Gorman44042b42021-06-28 19:43:08 -07005305 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags,
Mel Gorman387ba26f2021-04-29 23:01:45 -07005306 pcp, pcp_list);
Jesper Dangaard Brouerce76f9a2021-04-29 23:01:51 -07005307 if (unlikely(!page)) {
Mel Gorman387ba26f2021-04-29 23:01:45 -07005308 /* Try and get at least one page */
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005309 if (!nr_populated)
Mel Gorman387ba26f2021-04-29 23:01:45 -07005310 goto failed_irq;
5311 break;
5312 }
Mel Gorman3e230602021-06-28 19:41:50 -07005313 nr_account++;
Mel Gorman387ba26f2021-04-29 23:01:45 -07005314
5315 prep_new_page(page, 0, gfp, 0);
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005316 if (page_list)
5317 list_add(&page->lru, page_list);
5318 else
5319 page_array[nr_populated] = page;
5320 nr_populated++;
Mel Gorman387ba26f2021-04-29 23:01:45 -07005321 }
5322
Mel Gorman43c95bc2021-06-28 19:41:54 -07005323 local_unlock_irqrestore(&pagesets.lock, flags);
5324
Mel Gorman3e230602021-06-28 19:41:50 -07005325 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account);
5326 zone_statistics(ac.preferred_zoneref->zone, zone, nr_account);
Mel Gorman387ba26f2021-04-29 23:01:45 -07005327
Chuck Lever06147842021-07-14 21:26:52 -07005328out:
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005329 return nr_populated;
Mel Gorman387ba26f2021-04-29 23:01:45 -07005330
5331failed_irq:
Mel Gormandbbee9d2021-06-28 19:41:41 -07005332 local_unlock_irqrestore(&pagesets.lock, flags);
Mel Gorman387ba26f2021-04-29 23:01:45 -07005333
5334failed:
5335 page = __alloc_pages(gfp, 0, preferred_nid, nodemask);
5336 if (page) {
Mel Gorman0f87d9d2021-04-29 23:01:48 -07005337 if (page_list)
5338 list_add(&page->lru, page_list);
5339 else
5340 page_array[nr_populated] = page;
5341 nr_populated++;
Mel Gorman387ba26f2021-04-29 23:01:45 -07005342 }
5343
Chuck Lever06147842021-07-14 21:26:52 -07005344 goto out;
Mel Gorman387ba26f2021-04-29 23:01:45 -07005345}
5346EXPORT_SYMBOL_GPL(__alloc_pages_bulk);
5347
5348/*
Mel Gorman11e33f62009-06-16 15:31:57 -07005349 * This is the 'heart' of the zoned buddy allocator.
5350 */
Matthew Wilcox (Oracle)84172f42021-04-29 23:01:15 -07005351struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
Vlastimil Babka04ec6262017-07-06 15:40:03 -07005352 nodemask_t *nodemask)
Mel Gorman11e33f62009-06-16 15:31:57 -07005353{
Mel Gorman5bb1b162016-05-19 17:13:50 -07005354 struct page *page;
Mel Gormane6cbd7f2016-07-28 15:46:50 -07005355 unsigned int alloc_flags = ALLOC_WMARK_LOW;
Matthew Wilcox (Oracle)8e6a9302021-04-29 23:01:10 -07005356 gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
Mel Gorman9cd75552017-02-24 14:56:29 -08005357 struct alloc_context ac = { };
Mel Gorman682a3382016-05-19 17:13:30 -07005358
Michal Hockoc63ae432018-11-16 15:08:53 -08005359 /*
5360 * There are several places where we assume that the order value is sane
5361 * so bail out early if the request is out of bound.
5362 */
5363 if (unlikely(order >= MAX_ORDER)) {
Matthew Wilcox (Oracle)6e5e0f22021-04-29 23:01:13 -07005364 WARN_ON_ONCE(!(gfp & __GFP_NOWARN));
Michal Hockoc63ae432018-11-16 15:08:53 -08005365 return NULL;
5366 }
5367
Matthew Wilcox (Oracle)6e5e0f22021-04-29 23:01:13 -07005368 gfp &= gfp_allowed_mask;
Pavel Tatashinda6df1b2021-05-04 18:38:57 -07005369 /*
5370 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
5371 * resp. GFP_NOIO which has to be inherited for all allocation requests
5372 * from a particular context which has been marked by
Pavel Tatashin8e3560d2021-05-04 18:39:00 -07005373 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures
5374 * movable zones are not used during allocation.
Pavel Tatashinda6df1b2021-05-04 18:38:57 -07005375 */
5376 gfp = current_gfp_context(gfp);
Matthew Wilcox (Oracle)6e5e0f22021-04-29 23:01:13 -07005377 alloc_gfp = gfp;
5378 if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac,
Matthew Wilcox (Oracle)8e6a9302021-04-29 23:01:10 -07005379 &alloc_gfp, &alloc_flags))
Mel Gorman11e33f62009-06-16 15:31:57 -07005380 return NULL;
5381
Mel Gorman6bb15452018-12-28 00:35:41 -08005382 /*
5383 * Forbid the first pass from falling back to types that fragment
5384 * memory until all local zones are considered.
5385 */
Matthew Wilcox (Oracle)6e5e0f22021-04-29 23:01:13 -07005386 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp);
Mel Gorman6bb15452018-12-28 00:35:41 -08005387
Mel Gorman5117f452009-06-16 15:31:59 -07005388 /* First allocation attempt */
Matthew Wilcox (Oracle)8e6a9302021-04-29 23:01:10 -07005389 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
Mel Gorman4fcb0972016-05-19 17:14:01 -07005390 if (likely(page))
5391 goto out;
Andrew Morton91fbdc02015-02-11 15:25:04 -08005392
Pavel Tatashinda6df1b2021-05-04 18:38:57 -07005393 alloc_gfp = gfp;
Mel Gorman4fcb0972016-05-19 17:14:01 -07005394 ac.spread_dirty_pages = false;
Mel Gorman11e33f62009-06-16 15:31:57 -07005395
Mel Gorman47415262016-05-19 17:14:44 -07005396 /*
5397 * Restore the original nodemask if it was potentially replaced with
5398 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
5399 */
Mateusz Nosek97ce86f2020-04-01 21:09:53 -07005400 ac.nodemask = nodemask;
Vlastimil Babka16096c22017-01-24 15:18:35 -08005401
Matthew Wilcox (Oracle)8e6a9302021-04-29 23:01:10 -07005402 page = __alloc_pages_slowpath(alloc_gfp, order, &ac);
Xishi Qiu23f086f2015-02-11 15:25:07 -08005403
Mel Gorman4fcb0972016-05-19 17:14:01 -07005404out:
Matthew Wilcox (Oracle)6e5e0f22021-04-29 23:01:13 -07005405 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT) && page &&
5406 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) {
Vladimir Davydovc4159a72016-08-08 23:03:12 +03005407 __free_pages(page, order);
5408 page = NULL;
Vladimir Davydov49491482016-07-26 15:24:24 -07005409 }
5410
Matthew Wilcox (Oracle)8e6a9302021-04-29 23:01:10 -07005411 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
Mel Gorman4fcb0972016-05-19 17:14:01 -07005412
Mel Gorman11e33f62009-06-16 15:31:57 -07005413 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005414}
Matthew Wilcox (Oracle)84172f42021-04-29 23:01:15 -07005415EXPORT_SYMBOL(__alloc_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005416
Matthew Wilcox (Oracle)cc09cb12020-12-15 22:55:54 -05005417struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid,
5418 nodemask_t *nodemask)
5419{
5420 struct page *page = __alloc_pages(gfp | __GFP_COMP, order,
5421 preferred_nid, nodemask);
5422
5423 if (page && order > 1)
5424 prep_transhuge_page(page);
5425 return (struct folio *)page;
5426}
5427EXPORT_SYMBOL(__folio_alloc);
5428
Linus Torvalds1da177e2005-04-16 15:20:36 -07005429/*
Michal Hocko9ea9a682018-08-17 15:46:01 -07005430 * Common helper functions. Never use with __GFP_HIGHMEM because the returned
5431 * address cannot represent highmem pages. Use alloc_pages and then kmap if
5432 * you need to access high mem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005433 */
Harvey Harrison920c7a52008-02-04 22:29:26 -08005434unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005435{
Akinobu Mita945a1112009-09-21 17:01:47 -07005436 struct page *page;
5437
Michal Hocko9ea9a682018-08-17 15:46:01 -07005438 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005439 if (!page)
5440 return 0;
5441 return (unsigned long) page_address(page);
5442}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005443EXPORT_SYMBOL(__get_free_pages);
5444
Harvey Harrison920c7a52008-02-04 22:29:26 -08005445unsigned long get_zeroed_page(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005446{
Akinobu Mita945a1112009-09-21 17:01:47 -07005447 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005448}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005449EXPORT_SYMBOL(get_zeroed_page);
5450
Matthew Wilcox (Oracle)7f194fb2020-12-14 19:11:09 -08005451/**
5452 * __free_pages - Free pages allocated with alloc_pages().
5453 * @page: The page pointer returned from alloc_pages().
5454 * @order: The order of the allocation.
5455 *
5456 * This function can free multi-page allocations that are not compound
5457 * pages. It does not check that the @order passed in matches that of
5458 * the allocation, so it is easy to leak memory. Freeing more memory
5459 * than was allocated will probably emit a warning.
5460 *
5461 * If the last reference to this page is speculative, it will be released
5462 * by put_page() which only frees the first page of a non-compound
5463 * allocation. To prevent the remaining pages from being leaked, we free
5464 * the subsequent pages here. If you want to use the page's reference
5465 * count to decide when to free the allocation, you should allocate a
5466 * compound page, and use put_page() instead of __free_pages().
5467 *
5468 * Context: May be called in interrupt context or while holding a normal
5469 * spinlock, but not in NMI context or while holding a raw spinlock.
5470 */
Aaron Lu742aa7f2018-12-28 00:35:22 -08005471void __free_pages(struct page *page, unsigned int order)
5472{
5473 if (put_page_testzero(page))
5474 free_the_page(page, order);
Matthew Wilcox (Oracle)e320d302020-10-13 16:56:04 -07005475 else if (!PageHead(page))
5476 while (order-- > 0)
5477 free_the_page(page + (1 << order), order);
Aaron Lu742aa7f2018-12-28 00:35:22 -08005478}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005479EXPORT_SYMBOL(__free_pages);
5480
Harvey Harrison920c7a52008-02-04 22:29:26 -08005481void free_pages(unsigned long addr, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005482{
5483 if (addr != 0) {
Nick Piggin725d7042006-09-25 23:30:55 -07005484 VM_BUG_ON(!virt_addr_valid((void *)addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005485 __free_pages(virt_to_page((void *)addr), order);
5486 }
5487}
5488
5489EXPORT_SYMBOL(free_pages);
5490
Glauber Costa6a1a0d32012-12-18 14:22:00 -08005491/*
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005492 * Page Fragment:
5493 * An arbitrary-length arbitrary-offset area of memory which resides
5494 * within a 0 or higher order page. Multiple fragments within that page
5495 * are individually refcounted, in the page's reference counter.
5496 *
5497 * The page_frag functions below provide a simple allocation framework for
5498 * page fragments. This is used by the network stack and network device
5499 * drivers to provide a backing region of memory for use as either an
5500 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
5501 */
Alexander Duyck2976db82017-01-10 16:58:09 -08005502static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
5503 gfp_t gfp_mask)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005504{
5505 struct page *page = NULL;
5506 gfp_t gfp = gfp_mask;
5507
5508#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5509 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
5510 __GFP_NOMEMALLOC;
5511 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
5512 PAGE_FRAG_CACHE_MAX_ORDER);
5513 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
5514#endif
5515 if (unlikely(!page))
5516 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
5517
5518 nc->va = page ? page_address(page) : NULL;
5519
5520 return page;
5521}
5522
Alexander Duyck2976db82017-01-10 16:58:09 -08005523void __page_frag_cache_drain(struct page *page, unsigned int count)
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005524{
5525 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
5526
Aaron Lu742aa7f2018-12-28 00:35:22 -08005527 if (page_ref_sub_and_test(page, count))
5528 free_the_page(page, compound_order(page));
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005529}
Alexander Duyck2976db82017-01-10 16:58:09 -08005530EXPORT_SYMBOL(__page_frag_cache_drain);
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005531
Kevin Haob358e212021-02-04 18:56:35 +08005532void *page_frag_alloc_align(struct page_frag_cache *nc,
5533 unsigned int fragsz, gfp_t gfp_mask,
5534 unsigned int align_mask)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005535{
5536 unsigned int size = PAGE_SIZE;
5537 struct page *page;
5538 int offset;
5539
5540 if (unlikely(!nc->va)) {
5541refill:
Alexander Duyck2976db82017-01-10 16:58:09 -08005542 page = __page_frag_cache_refill(nc, gfp_mask);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005543 if (!page)
5544 return NULL;
5545
5546#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5547 /* if size can vary use size else just use PAGE_SIZE */
5548 size = nc->size;
5549#endif
5550 /* Even if we own the page, we do not use atomic_set().
5551 * This would break get_page_unless_zero() users.
5552 */
Alexander Duyck86447722019-02-15 14:44:12 -08005553 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005554
5555 /* reset page count bias and offset to start of new frag */
Michal Hocko2f064f32015-08-21 14:11:51 -07005556 nc->pfmemalloc = page_is_pfmemalloc(page);
Alexander Duyck86447722019-02-15 14:44:12 -08005557 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005558 nc->offset = size;
5559 }
5560
5561 offset = nc->offset - fragsz;
5562 if (unlikely(offset < 0)) {
5563 page = virt_to_page(nc->va);
5564
Joonsoo Kimfe896d12016-03-17 14:19:26 -07005565 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005566 goto refill;
5567
Dongli Zhangd8c19012020-11-15 12:10:29 -08005568 if (unlikely(nc->pfmemalloc)) {
5569 free_the_page(page, compound_order(page));
5570 goto refill;
5571 }
5572
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005573#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5574 /* if size can vary use size else just use PAGE_SIZE */
5575 size = nc->size;
5576#endif
5577 /* OK, page count is 0, we can safely set it */
Alexander Duyck86447722019-02-15 14:44:12 -08005578 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005579
5580 /* reset page count bias and offset to start of new frag */
Alexander Duyck86447722019-02-15 14:44:12 -08005581 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005582 offset = size - fragsz;
5583 }
5584
5585 nc->pagecnt_bias--;
Kevin Haob358e212021-02-04 18:56:35 +08005586 offset &= align_mask;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005587 nc->offset = offset;
5588
5589 return nc->va + offset;
5590}
Kevin Haob358e212021-02-04 18:56:35 +08005591EXPORT_SYMBOL(page_frag_alloc_align);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005592
5593/*
5594 * Frees a page fragment allocated out of either a compound or order 0 page.
5595 */
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005596void page_frag_free(void *addr)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005597{
5598 struct page *page = virt_to_head_page(addr);
5599
Aaron Lu742aa7f2018-12-28 00:35:22 -08005600 if (unlikely(put_page_testzero(page)))
5601 free_the_page(page, compound_order(page));
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005602}
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005603EXPORT_SYMBOL(page_frag_free);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005604
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005605static void *make_alloc_exact(unsigned long addr, unsigned int order,
5606 size_t size)
Andi Kleenee85c2e2011-05-11 15:13:34 -07005607{
5608 if (addr) {
5609 unsigned long alloc_end = addr + (PAGE_SIZE << order);
5610 unsigned long used = addr + PAGE_ALIGN(size);
5611
5612 split_page(virt_to_page((void *)addr), order);
5613 while (used < alloc_end) {
5614 free_page(used);
5615 used += PAGE_SIZE;
5616 }
5617 }
5618 return (void *)addr;
5619}
5620
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005621/**
5622 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
5623 * @size: the number of bytes to allocate
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005624 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005625 *
5626 * This function is similar to alloc_pages(), except that it allocates the
5627 * minimum number of pages to satisfy the request. alloc_pages() can only
5628 * allocate memory in power-of-two pages.
5629 *
5630 * This function is also limited by MAX_ORDER.
5631 *
5632 * Memory allocated by this function must be released by free_pages_exact().
Mike Rapoporta862f682019-03-05 15:48:42 -08005633 *
5634 * Return: pointer to the allocated area or %NULL in case of error.
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005635 */
5636void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
5637{
5638 unsigned int order = get_order(size);
5639 unsigned long addr;
5640
Miaohe Linba7f1b92021-11-05 13:40:15 -07005641 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM)))
5642 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM);
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005643
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005644 addr = __get_free_pages(gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07005645 return make_alloc_exact(addr, order, size);
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005646}
5647EXPORT_SYMBOL(alloc_pages_exact);
5648
5649/**
Andi Kleenee85c2e2011-05-11 15:13:34 -07005650 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
5651 * pages on a node.
Randy Dunlapb5e6ab52011-05-16 13:16:54 -07005652 * @nid: the preferred node ID where memory should be allocated
Andi Kleenee85c2e2011-05-11 15:13:34 -07005653 * @size: the number of bytes to allocate
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005654 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
Andi Kleenee85c2e2011-05-11 15:13:34 -07005655 *
5656 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
5657 * back.
Mike Rapoporta862f682019-03-05 15:48:42 -08005658 *
5659 * Return: pointer to the allocated area or %NULL in case of error.
Andi Kleenee85c2e2011-05-11 15:13:34 -07005660 */
Fabian Fredericke1931812014-08-06 16:04:59 -07005661void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
Andi Kleenee85c2e2011-05-11 15:13:34 -07005662{
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005663 unsigned int order = get_order(size);
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005664 struct page *p;
5665
Miaohe Linba7f1b92021-11-05 13:40:15 -07005666 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM)))
5667 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM);
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005668
5669 p = alloc_pages_node(nid, gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07005670 if (!p)
5671 return NULL;
5672 return make_alloc_exact((unsigned long)page_address(p), order, size);
5673}
Andi Kleenee85c2e2011-05-11 15:13:34 -07005674
5675/**
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005676 * free_pages_exact - release memory allocated via alloc_pages_exact()
5677 * @virt: the value returned by alloc_pages_exact.
5678 * @size: size of allocation, same value as passed to alloc_pages_exact().
5679 *
5680 * Release the memory allocated by a previous call to alloc_pages_exact.
5681 */
5682void free_pages_exact(void *virt, size_t size)
5683{
5684 unsigned long addr = (unsigned long)virt;
5685 unsigned long end = addr + PAGE_ALIGN(size);
5686
5687 while (addr < end) {
5688 free_page(addr);
5689 addr += PAGE_SIZE;
5690 }
5691}
5692EXPORT_SYMBOL(free_pages_exact);
5693
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005694/**
5695 * nr_free_zone_pages - count number of pages beyond high watermark
5696 * @offset: The zone index of the highest zone
5697 *
Mike Rapoporta862f682019-03-05 15:48:42 -08005698 * nr_free_zone_pages() counts the number of pages which are beyond the
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005699 * high watermark within all zones at or below a given zone index. For each
5700 * zone, the number of pages is calculated as:
mchehab@s-opensource.com0e056eb2017-03-30 17:11:36 -03005701 *
5702 * nr_free_zone_pages = managed_pages - high_pages
Mike Rapoporta862f682019-03-05 15:48:42 -08005703 *
5704 * Return: number of pages beyond high watermark.
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005705 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005706static unsigned long nr_free_zone_pages(int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005707{
Mel Gormandd1a2392008-04-28 02:12:17 -07005708 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07005709 struct zone *zone;
5710
Martin J. Blighe310fd42005-07-29 22:59:18 -07005711 /* Just pick one node, since fallback list is circular */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005712 unsigned long sum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005713
Mel Gorman0e884602008-04-28 02:12:14 -07005714 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005715
Mel Gorman54a6eb52008-04-28 02:12:16 -07005716 for_each_zone_zonelist(zone, z, zonelist, offset) {
Arun KS9705bea2018-12-28 00:34:24 -08005717 unsigned long size = zone_managed_pages(zone);
Mel Gorman41858962009-06-16 15:32:12 -07005718 unsigned long high = high_wmark_pages(zone);
Martin J. Blighe310fd42005-07-29 22:59:18 -07005719 if (size > high)
5720 sum += size - high;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005721 }
5722
5723 return sum;
5724}
5725
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005726/**
5727 * nr_free_buffer_pages - count number of pages beyond high watermark
5728 *
5729 * nr_free_buffer_pages() counts the number of pages which are beyond the high
5730 * watermark within ZONE_DMA and ZONE_NORMAL.
Mike Rapoporta862f682019-03-05 15:48:42 -08005731 *
5732 * Return: number of pages beyond high watermark within ZONE_DMA and
5733 * ZONE_NORMAL.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005734 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005735unsigned long nr_free_buffer_pages(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005736{
Al Viroaf4ca452005-10-21 02:55:38 -04005737 return nr_free_zone_pages(gfp_zone(GFP_USER));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005738}
Meelap Shahc2f1a552007-07-17 04:04:39 -07005739EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005740
Christoph Lameter08e0f6a2006-09-27 01:50:06 -07005741static inline void show_node(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005742{
Kirill A. Shutemove5adfff2012-12-11 16:00:29 -08005743 if (IS_ENABLED(CONFIG_NUMA))
Andy Whitcroft25ba77c2006-12-06 20:33:03 -08005744 printk("Node %d ", zone_to_nid(zone));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005745}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005746
Igor Redkod02bd272016-03-17 14:19:05 -07005747long si_mem_available(void)
5748{
5749 long available;
5750 unsigned long pagecache;
5751 unsigned long wmark_low = 0;
5752 unsigned long pages[NR_LRU_LISTS];
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005753 unsigned long reclaimable;
Igor Redkod02bd272016-03-17 14:19:05 -07005754 struct zone *zone;
5755 int lru;
5756
5757 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
Mel Gorman2f95ff92016-08-11 15:32:57 -07005758 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
Igor Redkod02bd272016-03-17 14:19:05 -07005759
5760 for_each_zone(zone)
Mel Gormana9214442018-12-28 00:35:44 -08005761 wmark_low += low_wmark_pages(zone);
Igor Redkod02bd272016-03-17 14:19:05 -07005762
5763 /*
5764 * Estimate the amount of memory available for userspace allocations,
5765 * without causing swapping.
5766 */
Michal Hockoc41f0122017-09-06 16:23:36 -07005767 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
Igor Redkod02bd272016-03-17 14:19:05 -07005768
5769 /*
5770 * Not all the page cache can be freed, otherwise the system will
5771 * start swapping. Assume at least half of the page cache, or the
5772 * low watermark worth of cache, needs to stay.
5773 */
5774 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
5775 pagecache -= min(pagecache / 2, wmark_low);
5776 available += pagecache;
5777
5778 /*
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005779 * Part of the reclaimable slab and other kernel memory consists of
5780 * items that are in use, and cannot be freed. Cap this estimate at the
5781 * low watermark.
Igor Redkod02bd272016-03-17 14:19:05 -07005782 */
Roman Gushchind42f3242020-08-06 23:20:39 -07005783 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) +
5784 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005785 available += reclaimable - min(reclaimable / 2, wmark_low);
Roman Gushchin034ebf62018-04-10 16:27:40 -07005786
Igor Redkod02bd272016-03-17 14:19:05 -07005787 if (available < 0)
5788 available = 0;
5789 return available;
5790}
5791EXPORT_SYMBOL_GPL(si_mem_available);
5792
Linus Torvalds1da177e2005-04-16 15:20:36 -07005793void si_meminfo(struct sysinfo *val)
5794{
Arun KSca79b0c2018-12-28 00:34:29 -08005795 val->totalram = totalram_pages();
Mel Gorman11fb9982016-07-28 15:46:20 -07005796 val->sharedram = global_node_page_state(NR_SHMEM);
Michal Hockoc41f0122017-09-06 16:23:36 -07005797 val->freeram = global_zone_page_state(NR_FREE_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005798 val->bufferram = nr_blockdev_pages();
Arun KSca79b0c2018-12-28 00:34:29 -08005799 val->totalhigh = totalhigh_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005800 val->freehigh = nr_free_highpages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005801 val->mem_unit = PAGE_SIZE;
5802}
5803
5804EXPORT_SYMBOL(si_meminfo);
5805
5806#ifdef CONFIG_NUMA
5807void si_meminfo_node(struct sysinfo *val, int nid)
5808{
Jiang Liucdd91a72013-07-03 15:03:27 -07005809 int zone_type; /* needs to be signed */
5810 unsigned long managed_pages = 0;
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005811 unsigned long managed_highpages = 0;
5812 unsigned long free_highpages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005813 pg_data_t *pgdat = NODE_DATA(nid);
5814
Jiang Liucdd91a72013-07-03 15:03:27 -07005815 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
Arun KS9705bea2018-12-28 00:34:24 -08005816 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
Jiang Liucdd91a72013-07-03 15:03:27 -07005817 val->totalram = managed_pages;
Mel Gorman11fb9982016-07-28 15:46:20 -07005818 val->sharedram = node_page_state(pgdat, NR_SHMEM);
Mel Gorman75ef7182016-07-28 15:45:24 -07005819 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005820#ifdef CONFIG_HIGHMEM
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005821 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
5822 struct zone *zone = &pgdat->node_zones[zone_type];
5823
5824 if (is_highmem(zone)) {
Arun KS9705bea2018-12-28 00:34:24 -08005825 managed_highpages += zone_managed_pages(zone);
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005826 free_highpages += zone_page_state(zone, NR_FREE_PAGES);
5827 }
5828 }
5829 val->totalhigh = managed_highpages;
5830 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005831#else
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005832 val->totalhigh = managed_highpages;
5833 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005834#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005835 val->mem_unit = PAGE_SIZE;
5836}
5837#endif
5838
David Rientjesddd588b2011-03-22 16:30:46 -07005839/*
David Rientjes7bf02ea2011-05-24 17:11:16 -07005840 * Determine whether the node should be displayed or not, depending on whether
5841 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
David Rientjesddd588b2011-03-22 16:30:46 -07005842 */
Michal Hocko9af744d2017-02-22 15:46:16 -08005843static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
David Rientjesddd588b2011-03-22 16:30:46 -07005844{
David Rientjesddd588b2011-03-22 16:30:46 -07005845 if (!(flags & SHOW_MEM_FILTER_NODES))
Michal Hocko9af744d2017-02-22 15:46:16 -08005846 return false;
David Rientjesddd588b2011-03-22 16:30:46 -07005847
Michal Hocko9af744d2017-02-22 15:46:16 -08005848 /*
5849 * no node mask - aka implicit memory numa policy. Do not bother with
5850 * the synchronization - read_mems_allowed_begin - because we do not
5851 * have to be precise here.
5852 */
5853 if (!nodemask)
5854 nodemask = &cpuset_current_mems_allowed;
5855
5856 return !node_isset(nid, *nodemask);
David Rientjesddd588b2011-03-22 16:30:46 -07005857}
5858
Linus Torvalds1da177e2005-04-16 15:20:36 -07005859#define K(x) ((x) << (PAGE_SHIFT-10))
5860
Rabin Vincent377e4f12012-12-11 16:00:24 -08005861static void show_migration_types(unsigned char type)
5862{
5863 static const char types[MIGRATE_TYPES] = {
5864 [MIGRATE_UNMOVABLE] = 'U',
Rabin Vincent377e4f12012-12-11 16:00:24 -08005865 [MIGRATE_MOVABLE] = 'M',
Vlastimil Babka475a2f92015-12-11 13:40:29 -08005866 [MIGRATE_RECLAIMABLE] = 'E',
5867 [MIGRATE_HIGHATOMIC] = 'H',
Rabin Vincent377e4f12012-12-11 16:00:24 -08005868#ifdef CONFIG_CMA
5869 [MIGRATE_CMA] = 'C',
5870#endif
Minchan Kim194159f2013-02-22 16:33:58 -08005871#ifdef CONFIG_MEMORY_ISOLATION
Rabin Vincent377e4f12012-12-11 16:00:24 -08005872 [MIGRATE_ISOLATE] = 'I',
Minchan Kim194159f2013-02-22 16:33:58 -08005873#endif
Rabin Vincent377e4f12012-12-11 16:00:24 -08005874 };
5875 char tmp[MIGRATE_TYPES + 1];
5876 char *p = tmp;
5877 int i;
5878
5879 for (i = 0; i < MIGRATE_TYPES; i++) {
5880 if (type & (1 << i))
5881 *p++ = types[i];
5882 }
5883
5884 *p = '\0';
Joe Perches1f84a182016-10-27 17:46:29 -07005885 printk(KERN_CONT "(%s) ", tmp);
Rabin Vincent377e4f12012-12-11 16:00:24 -08005886}
5887
Linus Torvalds1da177e2005-04-16 15:20:36 -07005888/*
5889 * Show free area list (used inside shift_scroll-lock stuff)
5890 * We also calculate the percentage fragmentation. We do this by counting the
5891 * memory on each free list with the exception of the first item on the list.
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005892 *
5893 * Bits in @filter:
5894 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
5895 * cpuset.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005896 */
Michal Hocko9af744d2017-02-22 15:46:16 -08005897void show_free_areas(unsigned int filter, nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005898{
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005899 unsigned long free_pcp = 0;
Jes Sorensenc7241912006-09-27 01:50:05 -07005900 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005901 struct zone *zone;
Mel Gorman599d0c92016-07-28 15:45:31 -07005902 pg_data_t *pgdat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005903
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005904 for_each_populated_zone(zone) {
Michal Hocko9af744d2017-02-22 15:46:16 -08005905 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005906 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005907
Konstantin Khlebnikov761b0672015-04-14 15:45:32 -07005908 for_each_online_cpu(cpu)
Mel Gorman28f836b2021-06-28 19:41:38 -07005909 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005910 }
5911
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07005912 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
5913 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
NeilBrown8d928902020-06-01 21:48:21 -07005914 " unevictable:%lu dirty:%lu writeback:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005915 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07005916 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
liuhailongeb2169c2021-09-02 14:53:01 -07005917 " kernel_misc_reclaimable:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005918 " free:%lu free_pcp:%lu free_cma:%lu\n",
Mel Gorman599d0c92016-07-28 15:45:31 -07005919 global_node_page_state(NR_ACTIVE_ANON),
5920 global_node_page_state(NR_INACTIVE_ANON),
5921 global_node_page_state(NR_ISOLATED_ANON),
5922 global_node_page_state(NR_ACTIVE_FILE),
5923 global_node_page_state(NR_INACTIVE_FILE),
5924 global_node_page_state(NR_ISOLATED_FILE),
5925 global_node_page_state(NR_UNEVICTABLE),
Mel Gorman11fb9982016-07-28 15:46:20 -07005926 global_node_page_state(NR_FILE_DIRTY),
5927 global_node_page_state(NR_WRITEBACK),
Roman Gushchind42f3242020-08-06 23:20:39 -07005928 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B),
5929 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B),
Mel Gorman50658e22016-07-28 15:46:14 -07005930 global_node_page_state(NR_FILE_MAPPED),
Mel Gorman11fb9982016-07-28 15:46:20 -07005931 global_node_page_state(NR_SHMEM),
Shakeel Buttf0c0c112020-12-14 19:07:17 -08005932 global_node_page_state(NR_PAGETABLE),
Michal Hockoc41f0122017-09-06 16:23:36 -07005933 global_zone_page_state(NR_BOUNCE),
liuhailongeb2169c2021-09-02 14:53:01 -07005934 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE),
Michal Hockoc41f0122017-09-06 16:23:36 -07005935 global_zone_page_state(NR_FREE_PAGES),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005936 free_pcp,
Michal Hockoc41f0122017-09-06 16:23:36 -07005937 global_zone_page_state(NR_FREE_CMA_PAGES));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005938
Mel Gorman599d0c92016-07-28 15:45:31 -07005939 for_each_online_pgdat(pgdat) {
Michal Hocko9af744d2017-02-22 15:46:16 -08005940 if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
Michal Hockoc02e50b2017-02-22 15:46:07 -08005941 continue;
5942
Mel Gorman599d0c92016-07-28 15:45:31 -07005943 printk("Node %d"
5944 " active_anon:%lukB"
5945 " inactive_anon:%lukB"
5946 " active_file:%lukB"
5947 " inactive_file:%lukB"
5948 " unevictable:%lukB"
5949 " isolated(anon):%lukB"
5950 " isolated(file):%lukB"
Mel Gorman50658e22016-07-28 15:46:14 -07005951 " mapped:%lukB"
Mel Gorman11fb9982016-07-28 15:46:20 -07005952 " dirty:%lukB"
5953 " writeback:%lukB"
5954 " shmem:%lukB"
5955#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5956 " shmem_thp: %lukB"
5957 " shmem_pmdmapped: %lukB"
5958 " anon_thp: %lukB"
5959#endif
5960 " writeback_tmp:%lukB"
Shakeel Butt991e7672020-08-06 23:21:37 -07005961 " kernel_stack:%lukB"
5962#ifdef CONFIG_SHADOW_CALL_STACK
5963 " shadow_call_stack:%lukB"
5964#endif
Shakeel Buttf0c0c112020-12-14 19:07:17 -08005965 " pagetables:%lukB"
Mel Gorman599d0c92016-07-28 15:45:31 -07005966 " all_unreclaimable? %s"
5967 "\n",
5968 pgdat->node_id,
5969 K(node_page_state(pgdat, NR_ACTIVE_ANON)),
5970 K(node_page_state(pgdat, NR_INACTIVE_ANON)),
5971 K(node_page_state(pgdat, NR_ACTIVE_FILE)),
5972 K(node_page_state(pgdat, NR_INACTIVE_FILE)),
5973 K(node_page_state(pgdat, NR_UNEVICTABLE)),
5974 K(node_page_state(pgdat, NR_ISOLATED_ANON)),
5975 K(node_page_state(pgdat, NR_ISOLATED_FILE)),
Mel Gorman50658e22016-07-28 15:46:14 -07005976 K(node_page_state(pgdat, NR_FILE_MAPPED)),
Mel Gorman11fb9982016-07-28 15:46:20 -07005977 K(node_page_state(pgdat, NR_FILE_DIRTY)),
5978 K(node_page_state(pgdat, NR_WRITEBACK)),
Alexander Polakov1f06b812017-04-07 16:04:45 -07005979 K(node_page_state(pgdat, NR_SHMEM)),
Mel Gorman11fb9982016-07-28 15:46:20 -07005980#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Muchun Song57b28472021-02-24 12:03:31 -08005981 K(node_page_state(pgdat, NR_SHMEM_THPS)),
Muchun Songa1528e22021-02-24 12:03:35 -08005982 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)),
Muchun Song69473e52021-02-24 12:03:23 -08005983 K(node_page_state(pgdat, NR_ANON_THPS)),
Mel Gorman11fb9982016-07-28 15:46:20 -07005984#endif
Mel Gorman11fb9982016-07-28 15:46:20 -07005985 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
Shakeel Butt991e7672020-08-06 23:21:37 -07005986 node_page_state(pgdat, NR_KERNEL_STACK_KB),
5987#ifdef CONFIG_SHADOW_CALL_STACK
5988 node_page_state(pgdat, NR_KERNEL_SCS_KB),
5989#endif
Shakeel Buttf0c0c112020-12-14 19:07:17 -08005990 K(node_page_state(pgdat, NR_PAGETABLE)),
Johannes Weinerc73322d2017-05-03 14:51:51 -07005991 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
5992 "yes" : "no");
Mel Gorman599d0c92016-07-28 15:45:31 -07005993 }
5994
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005995 for_each_populated_zone(zone) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005996 int i;
5997
Michal Hocko9af744d2017-02-22 15:46:16 -08005998 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005999 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07006000
6001 free_pcp = 0;
6002 for_each_online_cpu(cpu)
Mel Gorman28f836b2021-06-28 19:41:38 -07006003 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07006004
Linus Torvalds1da177e2005-04-16 15:20:36 -07006005 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07006006 printk(KERN_CONT
6007 "%s"
Linus Torvalds1da177e2005-04-16 15:20:36 -07006008 " free:%lukB"
Liangcai Fana6ea8b52021-11-05 13:40:37 -07006009 " boost:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07006010 " min:%lukB"
6011 " low:%lukB"
6012 " high:%lukB"
lijiazie47b3462019-11-30 17:55:21 -08006013 " reserved_highatomic:%luKB"
Minchan Kim71c799f2016-07-28 15:47:26 -07006014 " active_anon:%lukB"
6015 " inactive_anon:%lukB"
6016 " active_file:%lukB"
6017 " inactive_file:%lukB"
6018 " unevictable:%lukB"
Mel Gorman5a1c84b2016-07-28 15:47:31 -07006019 " writepending:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07006020 " present:%lukB"
Jiang Liu9feedc92012-12-12 13:52:12 -08006021 " managed:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07006022 " mlocked:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07006023 " bounce:%lukB"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07006024 " free_pcp:%lukB"
6025 " local_pcp:%ukB"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07006026 " free_cma:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07006027 "\n",
6028 zone->name,
Mel Gorman88f5acf2011-01-13 15:45:41 -08006029 K(zone_page_state(zone, NR_FREE_PAGES)),
Liangcai Fana6ea8b52021-11-05 13:40:37 -07006030 K(zone->watermark_boost),
Mel Gorman41858962009-06-16 15:32:12 -07006031 K(min_wmark_pages(zone)),
6032 K(low_wmark_pages(zone)),
6033 K(high_wmark_pages(zone)),
lijiazie47b3462019-11-30 17:55:21 -08006034 K(zone->nr_reserved_highatomic),
Minchan Kim71c799f2016-07-28 15:47:26 -07006035 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
6036 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
6037 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
6038 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
6039 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
Mel Gorman5a1c84b2016-07-28 15:47:31 -07006040 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
Linus Torvalds1da177e2005-04-16 15:20:36 -07006041 K(zone->present_pages),
Arun KS9705bea2018-12-28 00:34:24 -08006042 K(zone_managed_pages(zone)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07006043 K(zone_page_state(zone, NR_MLOCK)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07006044 K(zone_page_state(zone, NR_BOUNCE)),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07006045 K(free_pcp),
Mel Gorman28f836b2021-06-28 19:41:38 -07006046 K(this_cpu_read(zone->per_cpu_pageset->count)),
Minchan Kim33e077b2016-07-28 15:47:14 -07006047 K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07006048 printk("lowmem_reserve[]:");
6049 for (i = 0; i < MAX_NR_ZONES; i++)
Joe Perches1f84a182016-10-27 17:46:29 -07006050 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
6051 printk(KERN_CONT "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07006052 }
6053
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07006054 for_each_populated_zone(zone) {
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08006055 unsigned int order;
6056 unsigned long nr[MAX_ORDER], flags, total = 0;
Rabin Vincent377e4f12012-12-11 16:00:24 -08006057 unsigned char types[MAX_ORDER];
Linus Torvalds1da177e2005-04-16 15:20:36 -07006058
Michal Hocko9af744d2017-02-22 15:46:16 -08006059 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07006060 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006061 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07006062 printk(KERN_CONT "%s: ", zone->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006063
6064 spin_lock_irqsave(&zone->lock, flags);
6065 for (order = 0; order < MAX_ORDER; order++) {
Rabin Vincent377e4f12012-12-11 16:00:24 -08006066 struct free_area *area = &zone->free_area[order];
6067 int type;
6068
6069 nr[order] = area->nr_free;
Kirill Korotaev8f9de512006-06-23 02:03:50 -07006070 total += nr[order] << order;
Rabin Vincent377e4f12012-12-11 16:00:24 -08006071
6072 types[order] = 0;
6073 for (type = 0; type < MIGRATE_TYPES; type++) {
Dan Williamsb03641a2019-05-14 15:41:32 -07006074 if (!free_area_empty(area, type))
Rabin Vincent377e4f12012-12-11 16:00:24 -08006075 types[order] |= 1 << type;
6076 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006077 }
6078 spin_unlock_irqrestore(&zone->lock, flags);
Rabin Vincent377e4f12012-12-11 16:00:24 -08006079 for (order = 0; order < MAX_ORDER; order++) {
Joe Perches1f84a182016-10-27 17:46:29 -07006080 printk(KERN_CONT "%lu*%lukB ",
6081 nr[order], K(1UL) << order);
Rabin Vincent377e4f12012-12-11 16:00:24 -08006082 if (nr[order])
6083 show_migration_types(types[order]);
6084 }
Joe Perches1f84a182016-10-27 17:46:29 -07006085 printk(KERN_CONT "= %lukB\n", K(total));
Linus Torvalds1da177e2005-04-16 15:20:36 -07006086 }
6087
David Rientjes949f7ec2013-04-29 15:07:48 -07006088 hugetlb_show_meminfo();
6089
Mel Gorman11fb9982016-07-28 15:46:20 -07006090 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
Larry Woodmane6f36022008-02-04 22:29:30 -08006091
Linus Torvalds1da177e2005-04-16 15:20:36 -07006092 show_swap_cache_info();
6093}
6094
Mel Gorman19770b32008-04-28 02:12:18 -07006095static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
6096{
6097 zoneref->zone = zone;
6098 zoneref->zone_idx = zone_idx(zone);
6099}
6100
Linus Torvalds1da177e2005-04-16 15:20:36 -07006101/*
6102 * Builds allocation fallback zone lists.
Christoph Lameter1a932052006-01-06 00:11:16 -08006103 *
6104 * Add all populated zones of a node to the zonelist.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006105 */
Michal Hocko9d3be212017-09-06 16:20:30 -07006106static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006107{
Christoph Lameter1a932052006-01-06 00:11:16 -08006108 struct zone *zone;
Zhang Yanfeibc732f12013-07-08 16:00:06 -07006109 enum zone_type zone_type = MAX_NR_ZONES;
Michal Hocko9d3be212017-09-06 16:20:30 -07006110 int nr_zones = 0;
Christoph Lameter02a68a52006-01-06 00:11:18 -08006111
6112 do {
Christoph Lameter2f6726e2006-09-25 23:31:18 -07006113 zone_type--;
Christoph Lameter070f8032006-01-06 00:11:19 -08006114 zone = pgdat->node_zones + zone_type;
Mel Gorman6aa303d2016-09-01 16:14:55 -07006115 if (managed_zone(zone)) {
Michal Hocko9d3be212017-09-06 16:20:30 -07006116 zoneref_set_zone(zone, &zonerefs[nr_zones++]);
Christoph Lameter070f8032006-01-06 00:11:19 -08006117 check_highest_zone(zone_type);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006118 }
Christoph Lameter2f6726e2006-09-25 23:31:18 -07006119 } while (zone_type);
Zhang Yanfeibc732f12013-07-08 16:00:06 -07006120
Christoph Lameter070f8032006-01-06 00:11:19 -08006121 return nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006122}
6123
6124#ifdef CONFIG_NUMA
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006125
6126static int __parse_numa_zonelist_order(char *s)
6127{
Michal Hockoc9bff3e2017-09-06 16:20:13 -07006128 /*
Ingo Molnarf0953a12021-05-06 18:06:47 -07006129 * We used to support different zonelists modes but they turned
Michal Hockoc9bff3e2017-09-06 16:20:13 -07006130 * out to be just not useful. Let's keep the warning in place
6131 * if somebody still use the cmd line parameter so that we do
6132 * not fail it silently
6133 */
6134 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
6135 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006136 return -EINVAL;
6137 }
6138 return 0;
6139}
6140
Michal Hockoc9bff3e2017-09-06 16:20:13 -07006141char numa_zonelist_order[] = "Node";
6142
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006143/*
6144 * sysctl handler for numa_zonelist_order
6145 */
Joe Perchescccad5b2014-06-06 14:38:09 -07006146int numa_zonelist_order_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02006147 void *buffer, size_t *length, loff_t *ppos)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006148{
Christoph Hellwig32927392020-04-24 08:43:38 +02006149 if (write)
6150 return __parse_numa_zonelist_order(buffer);
6151 return proc_dostring(table, write, buffer, length, ppos);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006152}
6153
6154
Christoph Lameter62bc62a2009-06-16 15:32:15 -07006155#define MAX_NODE_LOAD (nr_online_nodes)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006156static int node_load[MAX_NUMNODES];
6157
Linus Torvalds1da177e2005-04-16 15:20:36 -07006158/**
Pavel Pisa4dc3b162005-05-01 08:59:25 -07006159 * find_next_best_node - find the next node that should appear in a given node's fallback list
Linus Torvalds1da177e2005-04-16 15:20:36 -07006160 * @node: node whose fallback list we're appending
6161 * @used_node_mask: nodemask_t of already used nodes
6162 *
6163 * We use a number of factors to determine which is the next node that should
6164 * appear on a given node's fallback list. The node should not have appeared
6165 * already in @node's fallback list, and it should be the next closest node
6166 * according to the distance array (which contains arbitrary distance values
6167 * from each node to each node in the system), and should also prefer nodes
6168 * with no CPUs, since presumably they'll have very little allocation pressure
6169 * on them otherwise.
Mike Rapoporta862f682019-03-05 15:48:42 -08006170 *
6171 * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006172 */
Dave Hansen79c28a42021-09-02 14:59:06 -07006173int find_next_best_node(int node, nodemask_t *used_node_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006174{
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01006175 int n, val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006176 int min_val = INT_MAX;
David Rientjes00ef2d22013-02-22 16:35:36 -08006177 int best_node = NUMA_NO_NODE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006178
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01006179 /* Use the local node if we haven't already */
6180 if (!node_isset(node, *used_node_mask)) {
6181 node_set(node, *used_node_mask);
6182 return node;
6183 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006184
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08006185 for_each_node_state(n, N_MEMORY) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006186
6187 /* Don't want a node to appear more than once */
6188 if (node_isset(n, *used_node_mask))
6189 continue;
6190
Linus Torvalds1da177e2005-04-16 15:20:36 -07006191 /* Use the distance array to find the distance */
6192 val = node_distance(node, n);
6193
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01006194 /* Penalize nodes under us ("prefer the next node") */
6195 val += (n < node);
6196
Linus Torvalds1da177e2005-04-16 15:20:36 -07006197 /* Give preference to headless and unused nodes */
Mateusz Nosekb6307492020-10-13 16:55:42 -07006198 if (!cpumask_empty(cpumask_of_node(n)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07006199 val += PENALTY_FOR_NODE_WITH_CPUS;
6200
6201 /* Slight preference for less loaded node */
6202 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
6203 val += node_load[n];
6204
6205 if (val < min_val) {
6206 min_val = val;
6207 best_node = n;
6208 }
6209 }
6210
6211 if (best_node >= 0)
6212 node_set(best_node, *used_node_mask);
6213
6214 return best_node;
6215}
6216
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006217
6218/*
6219 * Build zonelists ordered by node and zones within node.
6220 * This results in maximum locality--normal zone overflows into local
6221 * DMA zone, if any--but risks exhausting DMA zone.
6222 */
Michal Hocko9d3be212017-09-06 16:20:30 -07006223static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
6224 unsigned nr_nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006225{
Michal Hocko9d3be212017-09-06 16:20:30 -07006226 struct zoneref *zonerefs;
6227 int i;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006228
Michal Hocko9d3be212017-09-06 16:20:30 -07006229 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
6230
6231 for (i = 0; i < nr_nodes; i++) {
6232 int nr_zones;
6233
6234 pg_data_t *node = NODE_DATA(node_order[i]);
6235
6236 nr_zones = build_zonerefs_node(node, zonerefs);
6237 zonerefs += nr_zones;
6238 }
6239 zonerefs->zone = NULL;
6240 zonerefs->zone_idx = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006241}
6242
6243/*
Christoph Lameter523b9452007-10-16 01:25:37 -07006244 * Build gfp_thisnode zonelists
6245 */
6246static void build_thisnode_zonelists(pg_data_t *pgdat)
6247{
Michal Hocko9d3be212017-09-06 16:20:30 -07006248 struct zoneref *zonerefs;
6249 int nr_zones;
Christoph Lameter523b9452007-10-16 01:25:37 -07006250
Michal Hocko9d3be212017-09-06 16:20:30 -07006251 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
6252 nr_zones = build_zonerefs_node(pgdat, zonerefs);
6253 zonerefs += nr_zones;
6254 zonerefs->zone = NULL;
6255 zonerefs->zone_idx = 0;
Christoph Lameter523b9452007-10-16 01:25:37 -07006256}
6257
6258/*
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006259 * Build zonelists ordered by zone and nodes within zones.
6260 * This results in conserving DMA zone[s] until all Normal memory is
6261 * exhausted, but results in overflowing to remote node while memory
6262 * may still exist in local DMA zone.
6263 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006264
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006265static void build_zonelists(pg_data_t *pgdat)
6266{
Michal Hocko9d3be212017-09-06 16:20:30 -07006267 static int node_order[MAX_NUMNODES];
6268 int node, load, nr_nodes = 0;
Wei Yangd0ddf492020-06-03 15:59:05 -07006269 nodemask_t used_mask = NODE_MASK_NONE;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006270 int local_node, prev_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006271
6272 /* NUMA-aware ordering of nodes */
6273 local_node = pgdat->node_id;
Christoph Lameter62bc62a2009-06-16 15:32:15 -07006274 load = nr_online_nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006275 prev_node = local_node;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006276
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006277 memset(node_order, 0, sizeof(node_order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07006278 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
6279 /*
6280 * We don't want to pressure a particular node.
6281 * So adding penalty to the first node in same
6282 * distance group to make it round-robin.
6283 */
David Rientjes957f8222012-10-08 16:33:24 -07006284 if (node_distance(local_node, node) !=
6285 node_distance(local_node, prev_node))
Krupa Ramakrishnan54d032c2021-11-05 13:40:21 -07006286 node_load[node] += load;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006287
Michal Hocko9d3be212017-09-06 16:20:30 -07006288 node_order[nr_nodes++] = node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006289 prev_node = node;
6290 load--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006291 }
Christoph Lameter523b9452007-10-16 01:25:37 -07006292
Michal Hocko9d3be212017-09-06 16:20:30 -07006293 build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
Christoph Lameter523b9452007-10-16 01:25:37 -07006294 build_thisnode_zonelists(pgdat);
Bharata B Rao6cf25392021-11-05 13:40:18 -07006295 pr_info("Fallback order for Node %d: ", local_node);
6296 for (node = 0; node < nr_nodes; node++)
6297 pr_cont("%d ", node_order[node]);
6298 pr_cont("\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07006299}
6300
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07006301#ifdef CONFIG_HAVE_MEMORYLESS_NODES
6302/*
6303 * Return node id of node used for "local" allocations.
6304 * I.e., first node id of first zone in arg node's generic zonelist.
6305 * Used for initializing percpu 'numa_mem', which is used primarily
6306 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
6307 */
6308int local_memory_node(int node)
6309{
Mel Gormanc33d6c02016-05-19 17:14:10 -07006310 struct zoneref *z;
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07006311
Mel Gormanc33d6c02016-05-19 17:14:10 -07006312 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07006313 gfp_zone(GFP_KERNEL),
Mel Gormanc33d6c02016-05-19 17:14:10 -07006314 NULL);
Pavel Tatashinc1093b72018-08-21 21:53:32 -07006315 return zone_to_nid(z->zone);
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07006316}
6317#endif
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006318
Joonsoo Kim6423aa82016-08-10 16:27:49 -07006319static void setup_min_unmapped_ratio(void);
6320static void setup_min_slab_ratio(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006321#else /* CONFIG_NUMA */
6322
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006323static void build_zonelists(pg_data_t *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006324{
Christoph Lameter19655d32006-09-25 23:31:19 -07006325 int node, local_node;
Michal Hocko9d3be212017-09-06 16:20:30 -07006326 struct zoneref *zonerefs;
6327 int nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006328
6329 local_node = pgdat->node_id;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006330
Michal Hocko9d3be212017-09-06 16:20:30 -07006331 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
6332 nr_zones = build_zonerefs_node(pgdat, zonerefs);
6333 zonerefs += nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006334
Mel Gorman54a6eb52008-04-28 02:12:16 -07006335 /*
6336 * Now we build the zonelist so that it contains the zones
6337 * of all the other nodes.
6338 * We don't want to pressure a particular node, so when
6339 * building the zones for node N, we make sure that the
6340 * zones coming right after the local ones are those from
6341 * node N+1 (modulo N)
6342 */
6343 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
6344 if (!node_online(node))
6345 continue;
Michal Hocko9d3be212017-09-06 16:20:30 -07006346 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
6347 zonerefs += nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006348 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07006349 for (node = 0; node < local_node; node++) {
6350 if (!node_online(node))
6351 continue;
Michal Hocko9d3be212017-09-06 16:20:30 -07006352 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
6353 zonerefs += nr_zones;
Mel Gorman54a6eb52008-04-28 02:12:16 -07006354 }
6355
Michal Hocko9d3be212017-09-06 16:20:30 -07006356 zonerefs->zone = NULL;
6357 zonerefs->zone_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006358}
6359
6360#endif /* CONFIG_NUMA */
6361
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006362/*
6363 * Boot pageset table. One per cpu which is going to be used for all
6364 * zones and all nodes. The parameters will be set in such a way
6365 * that an item put on a list will immediately be handed over to
6366 * the buddy list. This is safe since pageset manipulation is done
6367 * with interrupts disabled.
6368 *
6369 * The boot_pagesets must be kept even after bootup is complete for
6370 * unused processors and/or zones. They do play a role for bootstrapping
6371 * hotplugged processors.
6372 *
6373 * zoneinfo_show() and maybe other functions do
6374 * not check if the processor is online before following the pageset pointer.
6375 * Other parts of the kernel may not check if the zone is available.
6376 */
Mel Gorman28f836b2021-06-28 19:41:38 -07006377static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats);
Vlastimil Babka952eaf82020-12-14 19:10:53 -08006378/* These effectively disable the pcplists in the boot pageset completely */
6379#define BOOT_PAGESET_HIGH 0
6380#define BOOT_PAGESET_BATCH 1
Mel Gorman28f836b2021-06-28 19:41:38 -07006381static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset);
6382static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats);
Johannes Weiner385386c2017-07-06 15:40:43 -07006383static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006384
Michal Hocko11cd8632017-09-06 16:20:34 -07006385static void __build_all_zonelists(void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006386{
Yasunori Goto68113782006-06-23 02:03:11 -07006387 int nid;
Michal Hockoafb6ebb2017-09-06 16:20:17 -07006388 int __maybe_unused cpu;
Jiang Liu9adb62a2012-07-31 16:43:28 -07006389 pg_data_t *self = data;
Michal Hockob93e0f32017-09-06 16:20:37 -07006390 static DEFINE_SPINLOCK(lock);
6391
6392 spin_lock(&lock);
Paul Jackson9276b1bc2006-12-06 20:31:48 -08006393
Bo Liu7f9cfb32009-08-18 14:11:19 -07006394#ifdef CONFIG_NUMA
6395 memset(node_load, 0, sizeof(node_load));
6396#endif
Jiang Liu9adb62a2012-07-31 16:43:28 -07006397
Wei Yangc1152582017-09-06 16:19:33 -07006398 /*
6399 * This node is hotadded and no memory is yet present. So just
6400 * building zonelists is fine - no need to touch other nodes.
6401 */
Jiang Liu9adb62a2012-07-31 16:43:28 -07006402 if (self && !node_online(self->node_id)) {
6403 build_zonelists(self);
Wei Yangc1152582017-09-06 16:19:33 -07006404 } else {
6405 for_each_online_node(nid) {
6406 pg_data_t *pgdat = NODE_DATA(nid);
Jiang Liu9adb62a2012-07-31 16:43:28 -07006407
Wei Yangc1152582017-09-06 16:19:33 -07006408 build_zonelists(pgdat);
6409 }
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006410
Michal Hockoafb6ebb2017-09-06 16:20:17 -07006411#ifdef CONFIG_HAVE_MEMORYLESS_NODES
Michal Hockoafb6ebb2017-09-06 16:20:17 -07006412 /*
6413 * We now know the "local memory node" for each node--
6414 * i.e., the node of the first zone in the generic zonelist.
6415 * Set up numa_mem percpu variable for on-line cpus. During
6416 * boot, only the boot cpu should be on-line; we'll init the
6417 * secondary cpus' numa_mem as they come on-line. During
6418 * node/memory hotplug, we'll fixup all on-line cpus.
6419 */
Michal Hockod9c9a0b2017-09-06 16:20:20 -07006420 for_each_online_cpu(cpu)
Michal Hockoafb6ebb2017-09-06 16:20:17 -07006421 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
Michal Hockoafb6ebb2017-09-06 16:20:17 -07006422#endif
Michal Hockod9c9a0b2017-09-06 16:20:20 -07006423 }
Michal Hockob93e0f32017-09-06 16:20:37 -07006424
6425 spin_unlock(&lock);
Michal Hockoafb6ebb2017-09-06 16:20:17 -07006426}
6427
6428static noinline void __init
6429build_all_zonelists_init(void)
6430{
6431 int cpu;
6432
6433 __build_all_zonelists(NULL);
6434
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006435 /*
6436 * Initialize the boot_pagesets that are going to be used
6437 * for bootstrapping processors. The real pagesets for
6438 * each zone will be allocated later when the per cpu
6439 * allocator is available.
6440 *
6441 * boot_pagesets are used also for bootstrapping offline
6442 * cpus if the system is already booted because the pagesets
6443 * are needed to initialize allocators on a specific cpu too.
6444 * F.e. the percpu allocator needs the page allocator which
6445 * needs the percpu allocator in order to allocate its pagesets
6446 * (a chicken-egg dilemma).
6447 */
Michal Hockoafb6ebb2017-09-06 16:20:17 -07006448 for_each_possible_cpu(cpu)
Mel Gorman28f836b2021-06-28 19:41:38 -07006449 per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu));
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006450
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08006451 mminit_verify_zonelist();
6452 cpuset_init_current_mems_allowed();
6453}
6454
Haicheng Li4eaf3f62010-05-24 14:32:52 -07006455/*
Haicheng Li4eaf3f62010-05-24 14:32:52 -07006456 * unless system_state == SYSTEM_BOOTING.
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08006457 *
Michal Hocko72675e12017-09-06 16:20:24 -07006458 * __ref due to call of __init annotated helper build_all_zonelists_init
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08006459 * [protected by SYSTEM_BOOTING].
Haicheng Li4eaf3f62010-05-24 14:32:52 -07006460 */
Michal Hocko72675e12017-09-06 16:20:24 -07006461void __ref build_all_zonelists(pg_data_t *pgdat)
Yasunori Goto68113782006-06-23 02:03:11 -07006462{
David Hildenbrand0a18e602020-08-06 23:25:27 -07006463 unsigned long vm_total_pages;
6464
Yasunori Goto68113782006-06-23 02:03:11 -07006465 if (system_state == SYSTEM_BOOTING) {
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08006466 build_all_zonelists_init();
Yasunori Goto68113782006-06-23 02:03:11 -07006467 } else {
Michal Hocko11cd8632017-09-06 16:20:34 -07006468 __build_all_zonelists(pgdat);
Yasunori Goto68113782006-06-23 02:03:11 -07006469 /* cpuset refresh routine should be here */
6470 }
David Hildenbrand56b94132020-08-06 23:25:30 -07006471 /* Get the number of free pages beyond high watermark in all zones. */
6472 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
Mel Gorman9ef9acb2007-10-16 01:25:54 -07006473 /*
6474 * Disable grouping by mobility if the number of pages in the
6475 * system is too low to allow the mechanism to work. It would be
6476 * more accurate, but expensive to check per-zone. This check is
6477 * made on memory-hotadd so a system can start with mobility
6478 * disabled and enable it later
6479 */
Mel Gormand9c23402007-10-16 01:26:01 -07006480 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
Mel Gorman9ef9acb2007-10-16 01:25:54 -07006481 page_group_by_mobility_disabled = 1;
6482 else
6483 page_group_by_mobility_disabled = 0;
6484
Alexey Dobriyance0725f2019-03-05 15:48:29 -08006485 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n",
Joe Perches756a0252016-03-17 14:19:47 -07006486 nr_online_nodes,
Joe Perches756a0252016-03-17 14:19:47 -07006487 page_group_by_mobility_disabled ? "off" : "on",
6488 vm_total_pages);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006489#ifdef CONFIG_NUMA
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006490 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006491#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07006492}
6493
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006494/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
6495static bool __meminit
6496overlap_memmap_init(unsigned long zone, unsigned long *pfn)
6497{
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006498 static struct memblock_region *r;
6499
6500 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
6501 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
Mike Rapoportcc6de162020-10-13 16:58:30 -07006502 for_each_mem_region(r) {
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006503 if (*pfn < memblock_region_memory_end_pfn(r))
6504 break;
6505 }
6506 }
6507 if (*pfn >= memblock_region_memory_base_pfn(r) &&
6508 memblock_is_mirror(r)) {
6509 *pfn = memblock_region_memory_end_pfn(r);
6510 return true;
6511 }
6512 }
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006513 return false;
6514}
6515
Linus Torvalds1da177e2005-04-16 15:20:36 -07006516/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07006517 * Initially all pages are reserved - free ones are freed
Mike Rapoportc6ffc5c2018-10-30 15:09:30 -07006518 * up by memblock_free_all() once the early boot process is
Linus Torvalds1da177e2005-04-16 15:20:36 -07006519 * done. Non-atomic initialization, single-pass.
David Hildenbrandd882c002020-10-15 20:08:19 -07006520 *
6521 * All aligned pageblocks are initialized to the specified migratetype
6522 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
6523 * zone stats (e.g., nr_isolate_pageblock) are touched.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006524 */
Baoquan Heab28cb62021-02-24 12:06:14 -08006525void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,
Baoquan Hedc2da7b2020-12-29 15:14:37 -08006526 unsigned long start_pfn, unsigned long zone_end_pfn,
David Hildenbrandd882c002020-10-15 20:08:19 -07006527 enum meminit_context context,
6528 struct vmem_altmap *altmap, int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006529{
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006530 unsigned long pfn, end_pfn = start_pfn + size;
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006531 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006532
Hugh Dickins22b31ee2009-01-06 14:40:09 -08006533 if (highest_memmap_pfn < end_pfn - 1)
6534 highest_memmap_pfn = end_pfn - 1;
6535
Alexander Duyck966cf442018-10-26 15:07:52 -07006536#ifdef CONFIG_ZONE_DEVICE
Dan Williams4b94ffd2016-01-15 16:56:22 -08006537 /*
6538 * Honor reservation requested by the driver for this ZONE_DEVICE
Alexander Duyck966cf442018-10-26 15:07:52 -07006539 * memory. We limit the total number of pages to initialize to just
6540 * those that might contain the memory mapping. We will defer the
6541 * ZONE_DEVICE page initialization until after we have released
6542 * the hotplug lock.
Dan Williams4b94ffd2016-01-15 16:56:22 -08006543 */
Alexander Duyck966cf442018-10-26 15:07:52 -07006544 if (zone == ZONE_DEVICE) {
6545 if (!altmap)
6546 return;
6547
6548 if (start_pfn == altmap->base_pfn)
6549 start_pfn += altmap->reserve;
6550 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
6551 }
6552#endif
Dan Williams4b94ffd2016-01-15 16:56:22 -08006553
David Hildenbrand948c4362020-02-03 17:33:59 -08006554 for (pfn = start_pfn; pfn < end_pfn; ) {
Dave Hansena2f3aa022007-01-10 23:15:30 -08006555 /*
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07006556 * There can be holes in boot-time mem_map[]s handed to this
6557 * function. They do not exist on hotplugged memory.
Dave Hansena2f3aa022007-01-10 23:15:30 -08006558 */
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006559 if (context == MEMINIT_EARLY) {
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006560 if (overlap_memmap_init(zone, &pfn))
6561 continue;
Baoquan Hedc2da7b2020-12-29 15:14:37 -08006562 if (defer_init(nid, pfn, zone_end_pfn))
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006563 break;
Dave Hansena2f3aa022007-01-10 23:15:30 -08006564 }
Mel Gormanac5d2532015-06-30 14:57:20 -07006565
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006566 page = pfn_to_page(pfn);
6567 __init_single_page(page, pfn, zone, nid);
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006568 if (context == MEMINIT_HOTPLUG)
Alexander Duyckd483da52018-10-26 15:07:48 -07006569 __SetPageReserved(page);
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006570
Mel Gormanac5d2532015-06-30 14:57:20 -07006571 /*
David Hildenbrandd882c002020-10-15 20:08:19 -07006572 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
6573 * such that unmovable allocations won't be scattered all
6574 * over the place during system boot.
Mel Gormanac5d2532015-06-30 14:57:20 -07006575 */
David Hildenbrand4eb29bd2020-10-15 20:08:15 -07006576 if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
David Hildenbrandd882c002020-10-15 20:08:19 -07006577 set_pageblock_migratetype(page, migratetype);
Michal Hocko9b6e63c2017-10-03 16:16:19 -07006578 cond_resched();
Mel Gormanac5d2532015-06-30 14:57:20 -07006579 }
David Hildenbrand948c4362020-02-03 17:33:59 -08006580 pfn++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006581 }
6582}
6583
Alexander Duyck966cf442018-10-26 15:07:52 -07006584#ifdef CONFIG_ZONE_DEVICE
Joao Martins46487e02022-01-14 14:04:18 -08006585static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
6586 unsigned long zone_idx, int nid,
6587 struct dev_pagemap *pgmap)
6588{
6589
6590 __init_single_page(page, pfn, zone_idx, nid);
6591
6592 /*
6593 * Mark page reserved as it will need to wait for onlining
6594 * phase for it to be fully associated with a zone.
6595 *
6596 * We can use the non-atomic __set_bit operation for setting
6597 * the flag as we are still initializing the pages.
6598 */
6599 __SetPageReserved(page);
6600
6601 /*
6602 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
6603 * and zone_device_data. It is a bug if a ZONE_DEVICE page is
6604 * ever freed or placed on a driver-private list.
6605 */
6606 page->pgmap = pgmap;
6607 page->zone_device_data = NULL;
6608
6609 /*
6610 * Mark the block movable so that blocks are reserved for
6611 * movable at startup. This will force kernel allocations
6612 * to reserve their blocks rather than leaking throughout
6613 * the address space during boot when many long-lived
6614 * kernel allocations are made.
6615 *
6616 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
6617 * because this is done early in section_activate()
6618 */
6619 if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
6620 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
6621 cond_resched();
6622 }
6623}
6624
Joao Martinsc4386bd2022-01-14 14:04:22 -08006625static void __ref memmap_init_compound(struct page *head,
6626 unsigned long head_pfn,
6627 unsigned long zone_idx, int nid,
6628 struct dev_pagemap *pgmap,
6629 unsigned long nr_pages)
6630{
6631 unsigned long pfn, end_pfn = head_pfn + nr_pages;
6632 unsigned int order = pgmap->vmemmap_shift;
6633
6634 __SetPageHead(head);
6635 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) {
6636 struct page *page = pfn_to_page(pfn);
6637
6638 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
6639 prep_compound_tail(head, pfn - head_pfn);
6640 set_page_count(page, 0);
6641
6642 /*
6643 * The first tail page stores compound_mapcount_ptr() and
6644 * compound_order() and the second tail page stores
6645 * compound_pincount_ptr(). Call prep_compound_head() after
6646 * the first and second tail pages have been initialized to
6647 * not have the data overwritten.
6648 */
6649 if (pfn == head_pfn + 2)
6650 prep_compound_head(head, order);
6651 }
6652}
6653
Alexander Duyck966cf442018-10-26 15:07:52 -07006654void __ref memmap_init_zone_device(struct zone *zone,
6655 unsigned long start_pfn,
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006656 unsigned long nr_pages,
Alexander Duyck966cf442018-10-26 15:07:52 -07006657 struct dev_pagemap *pgmap)
6658{
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006659 unsigned long pfn, end_pfn = start_pfn + nr_pages;
Alexander Duyck966cf442018-10-26 15:07:52 -07006660 struct pglist_data *pgdat = zone->zone_pgdat;
Christoph Hellwig514caf22019-06-26 14:27:13 +02006661 struct vmem_altmap *altmap = pgmap_altmap(pgmap);
Joao Martinsc4386bd2022-01-14 14:04:22 -08006662 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap);
Alexander Duyck966cf442018-10-26 15:07:52 -07006663 unsigned long zone_idx = zone_idx(zone);
6664 unsigned long start = jiffies;
6665 int nid = pgdat->node_id;
6666
Dan Williams46d945a2019-07-18 15:58:18 -07006667 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE))
Alexander Duyck966cf442018-10-26 15:07:52 -07006668 return;
6669
6670 /*
Mike Rapoport122e0932021-06-28 19:33:26 -07006671 * The call to memmap_init should have already taken care
Alexander Duyck966cf442018-10-26 15:07:52 -07006672 * of the pages reserved for the memmap, so we can just jump to
6673 * the end of that region and start processing the device pages.
6674 */
Christoph Hellwig514caf22019-06-26 14:27:13 +02006675 if (altmap) {
Alexander Duyck966cf442018-10-26 15:07:52 -07006676 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006677 nr_pages = end_pfn - start_pfn;
Alexander Duyck966cf442018-10-26 15:07:52 -07006678 }
6679
Joao Martinsc4386bd2022-01-14 14:04:22 -08006680 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {
Alexander Duyck966cf442018-10-26 15:07:52 -07006681 struct page *page = pfn_to_page(pfn);
6682
Joao Martins46487e02022-01-14 14:04:18 -08006683 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
Joao Martinsc4386bd2022-01-14 14:04:22 -08006684
6685 if (pfns_per_compound == 1)
6686 continue;
6687
6688 memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
6689 pfns_per_compound);
Alexander Duyck966cf442018-10-26 15:07:52 -07006690 }
6691
Christoph Hellwigfdc029b2019-08-18 11:05:55 +02006692 pr_info("%s initialised %lu pages in %ums\n", __func__,
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006693 nr_pages, jiffies_to_msecs(jiffies - start));
Alexander Duyck966cf442018-10-26 15:07:52 -07006694}
6695
6696#endif
Andi Kleen1e548de2008-02-04 22:29:26 -08006697static void __meminit zone_init_free_lists(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006698{
Mel Gorman7aeb09f2014-06-04 16:10:21 -07006699 unsigned int order, t;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07006700 for_each_migratetype_order(order, t) {
6701 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006702 zone->free_area[order].nr_free = 0;
6703 }
6704}
6705
Mike Rapoport0740a502021-03-12 21:07:12 -08006706/*
6707 * Only struct pages that correspond to ranges defined by memblock.memory
6708 * are zeroed and initialized by going through __init_single_page() during
Mike Rapoport122e0932021-06-28 19:33:26 -07006709 * memmap_init_zone_range().
Mike Rapoport0740a502021-03-12 21:07:12 -08006710 *
6711 * But, there could be struct pages that correspond to holes in
6712 * memblock.memory. This can happen because of the following reasons:
6713 * - physical memory bank size is not necessarily the exact multiple of the
6714 * arbitrary section size
6715 * - early reserved memory may not be listed in memblock.memory
6716 * - memory layouts defined with memmap= kernel parameter may not align
6717 * nicely with memmap sections
6718 *
6719 * Explicitly initialize those struct pages so that:
6720 * - PG_Reserved is set
6721 * - zone and node links point to zone and node that span the page if the
6722 * hole is in the middle of a zone
6723 * - zone and node links point to adjacent zone/node if the hole falls on
6724 * the zone boundary; the pages in such holes will be prepended to the
6725 * zone/node above the hole except for the trailing pages in the last
6726 * section that will be appended to the zone/node below.
6727 */
Mike Rapoport122e0932021-06-28 19:33:26 -07006728static void __init init_unavailable_range(unsigned long spfn,
6729 unsigned long epfn,
6730 int zone, int node)
Mike Rapoport0740a502021-03-12 21:07:12 -08006731{
6732 unsigned long pfn;
6733 u64 pgcnt = 0;
6734
6735 for (pfn = spfn; pfn < epfn; pfn++) {
6736 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
6737 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
6738 + pageblock_nr_pages - 1;
6739 continue;
6740 }
6741 __init_single_page(pfn_to_page(pfn), pfn, zone, node);
6742 __SetPageReserved(pfn_to_page(pfn));
6743 pgcnt++;
6744 }
6745
Mike Rapoport122e0932021-06-28 19:33:26 -07006746 if (pgcnt)
6747 pr_info("On node %d, zone %s: %lld pages in unavailable ranges",
6748 node, zone_names[zone], pgcnt);
Mike Rapoport0740a502021-03-12 21:07:12 -08006749}
Mike Rapoport0740a502021-03-12 21:07:12 -08006750
Mike Rapoport122e0932021-06-28 19:33:26 -07006751static void __init memmap_init_zone_range(struct zone *zone,
6752 unsigned long start_pfn,
6753 unsigned long end_pfn,
6754 unsigned long *hole_pfn)
Pavel Tatashindfb3ccd2018-10-26 15:09:32 -07006755{
Baoquan He3256ff82021-02-24 12:06:17 -08006756 unsigned long zone_start_pfn = zone->zone_start_pfn;
6757 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
Mike Rapoport122e0932021-06-28 19:33:26 -07006758 int nid = zone_to_nid(zone), zone_id = zone_idx(zone);
6759
6760 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);
6761 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);
6762
6763 if (start_pfn >= end_pfn)
6764 return;
6765
6766 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
6767 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
6768
6769 if (*hole_pfn < start_pfn)
6770 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
6771
6772 *hole_pfn = end_pfn;
6773}
6774
6775static void __init memmap_init(void)
6776{
Baoquan He73a6e472020-06-03 15:57:55 -07006777 unsigned long start_pfn, end_pfn;
Mike Rapoport122e0932021-06-28 19:33:26 -07006778 unsigned long hole_pfn = 0;
Nico Pacheb3460752021-09-02 14:58:08 -07006779 int i, j, zone_id = 0, nid;
Baoquan He73a6e472020-06-03 15:57:55 -07006780
Mike Rapoport122e0932021-06-28 19:33:26 -07006781 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
6782 struct pglist_data *node = NODE_DATA(nid);
Baoquan He73a6e472020-06-03 15:57:55 -07006783
Mike Rapoport122e0932021-06-28 19:33:26 -07006784 for (j = 0; j < MAX_NR_ZONES; j++) {
6785 struct zone *zone = node->node_zones + j;
Mike Rapoport0740a502021-03-12 21:07:12 -08006786
Mike Rapoport122e0932021-06-28 19:33:26 -07006787 if (!populated_zone(zone))
6788 continue;
6789
6790 memmap_init_zone_range(zone, start_pfn, end_pfn,
6791 &hole_pfn);
6792 zone_id = j;
6793 }
Baoquan He73a6e472020-06-03 15:57:55 -07006794 }
Mike Rapoport0740a502021-03-12 21:07:12 -08006795
6796#ifdef CONFIG_SPARSEMEM
6797 /*
Mike Rapoport122e0932021-06-28 19:33:26 -07006798 * Initialize the memory map for hole in the range [memory_end,
6799 * section_end].
6800 * Append the pages in this hole to the highest zone in the last
6801 * node.
6802 * The call to init_unavailable_range() is outside the ifdef to
6803 * silence the compiler warining about zone_id set but not used;
6804 * for FLATMEM it is a nop anyway
Mike Rapoport0740a502021-03-12 21:07:12 -08006805 */
Mike Rapoport122e0932021-06-28 19:33:26 -07006806 end_pfn = round_up(end_pfn, PAGES_PER_SECTION);
Mike Rapoport0740a502021-03-12 21:07:12 -08006807 if (hole_pfn < end_pfn)
Mike Rapoport0740a502021-03-12 21:07:12 -08006808#endif
Mike Rapoport122e0932021-06-28 19:33:26 -07006809 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);
Pavel Tatashindfb3ccd2018-10-26 15:09:32 -07006810}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006811
Mike Rapoportc803b3c2021-09-02 14:58:02 -07006812void __init *memmap_alloc(phys_addr_t size, phys_addr_t align,
6813 phys_addr_t min_addr, int nid, bool exact_nid)
6814{
6815 void *ptr;
6816
6817 if (exact_nid)
6818 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr,
6819 MEMBLOCK_ALLOC_ACCESSIBLE,
6820 nid);
6821 else
6822 ptr = memblock_alloc_try_nid_raw(size, align, min_addr,
6823 MEMBLOCK_ALLOC_ACCESSIBLE,
6824 nid);
6825
6826 if (ptr && size > 0)
6827 page_init_poison(ptr, size);
6828
6829 return ptr;
6830}
6831
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006832static int zone_batchsize(struct zone *zone)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006833{
David Howells3a6be872009-05-06 16:03:03 -07006834#ifdef CONFIG_MMU
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006835 int batch;
6836
6837 /*
Mel Gormanb92ca182021-06-28 19:42:12 -07006838 * The number of pages to batch allocate is either ~0.1%
6839 * of the zone or 1MB, whichever is smaller. The batch
6840 * size is striking a balance between allocation latency
6841 * and zone lock contention.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006842 */
Mel Gormanb92ca182021-06-28 19:42:12 -07006843 batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006844 batch /= 4; /* We effectively *= 4 below */
6845 if (batch < 1)
6846 batch = 1;
6847
6848 /*
Nick Piggin0ceaacc2005-12-04 13:55:25 +11006849 * Clamp the batch to a 2^n - 1 value. Having a power
6850 * of 2 value was found to be more likely to have
6851 * suboptimal cache aliasing properties in some cases.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006852 *
Nick Piggin0ceaacc2005-12-04 13:55:25 +11006853 * For example if 2 tasks are alternately allocating
6854 * batches of pages, one task can end up with a lot
6855 * of pages of one half of the possible page colors
6856 * and the other with pages of the other colors.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006857 */
David Howells91552032009-05-06 16:03:02 -07006858 batch = rounddown_pow_of_two(batch + batch/2) - 1;
Seth, Rohitba56e912005-10-29 18:15:47 -07006859
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006860 return batch;
David Howells3a6be872009-05-06 16:03:03 -07006861
6862#else
6863 /* The deferral and batching of frees should be suppressed under NOMMU
6864 * conditions.
6865 *
6866 * The problem is that NOMMU needs to be able to allocate large chunks
6867 * of contiguous memory as there's no hardware page translation to
6868 * assemble apparent contiguous memory from discontiguous pages.
6869 *
6870 * Queueing large contiguous runs of pages for batching, however,
6871 * causes the pages to actually be freed in smaller chunks. As there
6872 * can be a significant delay between the individual batches being
6873 * recycled, this leads to the once large chunks of space being
6874 * fragmented and becoming unavailable for high-order allocations.
6875 */
6876 return 0;
6877#endif
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006878}
6879
Mel Gorman04f8cfe2021-06-28 19:42:15 -07006880static int zone_highsize(struct zone *zone, int batch, int cpu_online)
Mel Gormanb92ca182021-06-28 19:42:12 -07006881{
6882#ifdef CONFIG_MMU
6883 int high;
Mel Gorman203c06e2021-06-28 19:43:11 -07006884 int nr_split_cpus;
Mel Gorman74f44822021-06-28 19:42:24 -07006885 unsigned long total_pages;
6886
6887 if (!percpu_pagelist_high_fraction) {
6888 /*
6889 * By default, the high value of the pcp is based on the zone
6890 * low watermark so that if they are full then background
6891 * reclaim will not be started prematurely.
6892 */
6893 total_pages = low_wmark_pages(zone);
6894 } else {
6895 /*
6896 * If percpu_pagelist_high_fraction is configured, the high
6897 * value is based on a fraction of the managed pages in the
6898 * zone.
6899 */
6900 total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction;
6901 }
Mel Gormanb92ca182021-06-28 19:42:12 -07006902
6903 /*
Mel Gorman74f44822021-06-28 19:42:24 -07006904 * Split the high value across all online CPUs local to the zone. Note
6905 * that early in boot that CPUs may not be online yet and that during
6906 * CPU hotplug that the cpumask is not yet updated when a CPU is being
Mel Gorman203c06e2021-06-28 19:43:11 -07006907 * onlined. For memory nodes that have no CPUs, split pcp->high across
6908 * all online CPUs to mitigate the risk that reclaim is triggered
6909 * prematurely due to pages stored on pcp lists.
Mel Gormanb92ca182021-06-28 19:42:12 -07006910 */
Mel Gorman203c06e2021-06-28 19:43:11 -07006911 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online;
6912 if (!nr_split_cpus)
6913 nr_split_cpus = num_online_cpus();
6914 high = total_pages / nr_split_cpus;
Mel Gormanb92ca182021-06-28 19:42:12 -07006915
6916 /*
6917 * Ensure high is at least batch*4. The multiple is based on the
6918 * historical relationship between high and batch.
6919 */
6920 high = max(high, batch << 2);
6921
6922 return high;
6923#else
6924 return 0;
6925#endif
6926}
6927
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006928/*
Vlastimil Babka5c3ad2e2020-12-14 19:10:50 -08006929 * pcp->high and pcp->batch values are related and generally batch is lower
6930 * than high. They are also related to pcp->count such that count is lower
6931 * than high, and as soon as it reaches high, the pcplist is flushed.
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006932 *
Vlastimil Babka5c3ad2e2020-12-14 19:10:50 -08006933 * However, guaranteeing these relations at all times would require e.g. write
6934 * barriers here but also careful usage of read barriers at the read side, and
6935 * thus be prone to error and bad for performance. Thus the update only prevents
6936 * store tearing. Any new users of pcp->batch and pcp->high should ensure they
6937 * can cope with those fields changing asynchronously, and fully trust only the
6938 * pcp->count field on the local CPU with interrupts disabled.
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006939 *
6940 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
6941 * outside of boot time (or some other assurance that no concurrent updaters
6942 * exist).
6943 */
6944static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
6945 unsigned long batch)
6946{
Vlastimil Babka5c3ad2e2020-12-14 19:10:50 -08006947 WRITE_ONCE(pcp->batch, batch);
6948 WRITE_ONCE(pcp->high, high);
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006949}
6950
Mel Gorman28f836b2021-06-28 19:41:38 -07006951static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats)
Christoph Lameter2caaad42005-06-21 17:15:00 -07006952{
Mel Gorman44042b42021-06-28 19:43:08 -07006953 int pindex;
Christoph Lameter2caaad42005-06-21 17:15:00 -07006954
Mel Gorman28f836b2021-06-28 19:41:38 -07006955 memset(pcp, 0, sizeof(*pcp));
6956 memset(pzstats, 0, sizeof(*pzstats));
Magnus Damm1c6fe942005-10-26 01:58:59 -07006957
Mel Gorman44042b42021-06-28 19:43:08 -07006958 for (pindex = 0; pindex < NR_PCP_LISTS; pindex++)
6959 INIT_LIST_HEAD(&pcp->lists[pindex]);
Christoph Lameter2caaad42005-06-21 17:15:00 -07006960
Vlastimil Babka69a83962020-12-14 19:10:47 -08006961 /*
6962 * Set batch and high values safe for a boot pageset. A true percpu
6963 * pageset's initialization will update them subsequently. Here we don't
6964 * need to be as careful as pageset_update() as nobody can access the
6965 * pageset yet.
6966 */
Vlastimil Babka952eaf82020-12-14 19:10:53 -08006967 pcp->high = BOOT_PAGESET_HIGH;
6968 pcp->batch = BOOT_PAGESET_BATCH;
Mel Gorman3b12e7e2021-06-28 19:42:18 -07006969 pcp->free_factor = 0;
Cody P Schafer88c90db2013-07-03 15:01:35 -07006970}
6971
Zou Wei3b1f3652020-12-14 19:11:12 -08006972static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high,
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08006973 unsigned long batch)
6974{
Mel Gorman28f836b2021-06-28 19:41:38 -07006975 struct per_cpu_pages *pcp;
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08006976 int cpu;
6977
6978 for_each_possible_cpu(cpu) {
Mel Gorman28f836b2021-06-28 19:41:38 -07006979 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
6980 pageset_update(pcp, high, batch);
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08006981 }
6982}
6983
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006984/*
Vlastimil Babka0a8b4f12020-12-14 19:10:43 -08006985 * Calculate and set new high and batch values for all per-cpu pagesets of a
Mel Gormanbbbecb32021-06-28 19:42:09 -07006986 * zone based on the zone's size.
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006987 */
Mel Gorman04f8cfe2021-06-28 19:42:15 -07006988static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online)
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006989{
Mel Gormanb92ca182021-06-28 19:42:12 -07006990 int new_high, new_batch;
Vlastimil Babka7115ac62020-12-14 19:10:40 -08006991
Mel Gormanb92ca182021-06-28 19:42:12 -07006992 new_batch = max(1, zone_batchsize(zone));
Mel Gorman04f8cfe2021-06-28 19:42:15 -07006993 new_high = zone_highsize(zone, new_batch, cpu_online);
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006994
Vlastimil Babka952eaf82020-12-14 19:10:53 -08006995 if (zone->pageset_high == new_high &&
6996 zone->pageset_batch == new_batch)
6997 return;
6998
6999 zone->pageset_high = new_high;
7000 zone->pageset_batch = new_batch;
7001
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08007002 __zone_set_pageset_high_and_batch(zone, new_high, new_batch);
Cody P Schafer169f6c12013-07-03 15:01:41 -07007003}
7004
Michal Hocko72675e12017-09-06 16:20:24 -07007005void __meminit setup_zone_pageset(struct zone *zone)
Wu Fengguang319774e2010-05-24 14:32:49 -07007006{
7007 int cpu;
Vlastimil Babka0a8b4f12020-12-14 19:10:43 -08007008
Mel Gorman28f836b2021-06-28 19:41:38 -07007009 /* Size may be 0 on !SMP && !NUMA */
7010 if (sizeof(struct per_cpu_zonestat) > 0)
7011 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat);
7012
7013 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages);
Vlastimil Babka0a8b4f12020-12-14 19:10:43 -08007014 for_each_possible_cpu(cpu) {
Mel Gorman28f836b2021-06-28 19:41:38 -07007015 struct per_cpu_pages *pcp;
7016 struct per_cpu_zonestat *pzstats;
7017
7018 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
7019 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
7020 per_cpu_pages_init(pcp, pzstats);
Vlastimil Babka0a8b4f12020-12-14 19:10:43 -08007021 }
7022
Mel Gorman04f8cfe2021-06-28 19:42:15 -07007023 zone_set_pageset_high_and_batch(zone, 0);
Wu Fengguang319774e2010-05-24 14:32:49 -07007024}
7025
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07007026/*
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09007027 * Allocate per cpu pagesets and initialize them.
7028 * Before this call only boot pagesets were available.
Christoph Lameter2caaad42005-06-21 17:15:00 -07007029 */
Al Viro78d99552005-12-15 09:18:25 +00007030void __init setup_per_cpu_pageset(void)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07007031{
Mel Gormanb4911ea2016-08-04 15:31:49 -07007032 struct pglist_data *pgdat;
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09007033 struct zone *zone;
Sandipan Dasb418a0f2020-06-03 15:59:11 -07007034 int __maybe_unused cpu;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07007035
Wu Fengguang319774e2010-05-24 14:32:49 -07007036 for_each_populated_zone(zone)
7037 setup_zone_pageset(zone);
Mel Gormanb4911ea2016-08-04 15:31:49 -07007038
Sandipan Dasb418a0f2020-06-03 15:59:11 -07007039#ifdef CONFIG_NUMA
7040 /*
7041 * Unpopulated zones continue using the boot pagesets.
7042 * The numa stats for these pagesets need to be reset.
7043 * Otherwise, they will end up skewing the stats of
7044 * the nodes these zones are associated with.
7045 */
7046 for_each_possible_cpu(cpu) {
Mel Gorman28f836b2021-06-28 19:41:38 -07007047 struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu);
Mel Gormanf19298b2021-06-28 19:41:44 -07007048 memset(pzstats->vm_numa_event, 0,
7049 sizeof(pzstats->vm_numa_event));
Sandipan Dasb418a0f2020-06-03 15:59:11 -07007050 }
7051#endif
7052
Mel Gormanb4911ea2016-08-04 15:31:49 -07007053 for_each_online_pgdat(pgdat)
7054 pgdat->per_cpu_nodestats =
7055 alloc_percpu(struct per_cpu_nodestat);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07007056}
7057
Matt Tolentinoc09b4242006-01-17 07:03:44 +01007058static __meminit void zone_pcp_init(struct zone *zone)
Dave Hansened8ece22005-10-29 18:16:50 -07007059{
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09007060 /*
7061 * per cpu subsystem is not up at this point. The following code
7062 * relies on the ability of the linker to provide the
7063 * offset of a (static) per cpu variable into the per cpu area.
7064 */
Mel Gorman28f836b2021-06-28 19:41:38 -07007065 zone->per_cpu_pageset = &boot_pageset;
7066 zone->per_cpu_zonestats = &boot_zonestats;
Vlastimil Babka952eaf82020-12-14 19:10:53 -08007067 zone->pageset_high = BOOT_PAGESET_HIGH;
7068 zone->pageset_batch = BOOT_PAGESET_BATCH;
Dave Hansened8ece22005-10-29 18:16:50 -07007069
Xishi Qiub38a8722013-11-12 15:07:20 -08007070 if (populated_zone(zone))
Heiner Kallweit9660eca2021-06-28 19:41:31 -07007071 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name,
7072 zone->present_pages, zone_batchsize(zone));
Dave Hansened8ece22005-10-29 18:16:50 -07007073}
7074
Michal Hockodc0bbf32017-07-06 15:37:35 -07007075void __meminit init_currently_empty_zone(struct zone *zone,
Yasunori Goto718127c2006-06-23 02:03:10 -07007076 unsigned long zone_start_pfn,
Yaowei Baib171e402015-11-05 18:47:06 -08007077 unsigned long size)
Dave Hansened8ece22005-10-29 18:16:50 -07007078{
7079 struct pglist_data *pgdat = zone->zone_pgdat;
Wei Yang8f416832018-11-30 14:09:07 -08007080 int zone_idx = zone_idx(zone) + 1;
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07007081
Wei Yang8f416832018-11-30 14:09:07 -08007082 if (zone_idx > pgdat->nr_zones)
7083 pgdat->nr_zones = zone_idx;
Dave Hansened8ece22005-10-29 18:16:50 -07007084
Dave Hansened8ece22005-10-29 18:16:50 -07007085 zone->zone_start_pfn = zone_start_pfn;
7086
Mel Gorman708614e2008-07-23 21:26:51 -07007087 mminit_dprintk(MMINIT_TRACE, "memmap_init",
7088 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
7089 pgdat->node_id,
7090 (unsigned long)zone_idx(zone),
7091 zone_start_pfn, (zone_start_pfn + size));
7092
Andi Kleen1e548de2008-02-04 22:29:26 -08007093 zone_init_free_lists(zone);
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07007094 zone->initialized = 1;
Dave Hansened8ece22005-10-29 18:16:50 -07007095}
7096
Mel Gormanc7132162006-09-27 01:49:43 -07007097/**
Mel Gormanc7132162006-09-27 01:49:43 -07007098 * get_pfn_range_for_nid - Return the start and end page frames for a node
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007099 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
7100 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
7101 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
Mel Gormanc7132162006-09-27 01:49:43 -07007102 *
7103 * It returns the start and end page frame of a node based on information
Zhang Zhen7d018172014-06-04 16:10:53 -07007104 * provided by memblock_set_node(). If called for a node
Mel Gormanc7132162006-09-27 01:49:43 -07007105 * with no available memory, a warning is printed and the start and end
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007106 * PFNs will be 0.
Mel Gormanc7132162006-09-27 01:49:43 -07007107 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08007108void __init get_pfn_range_for_nid(unsigned int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07007109 unsigned long *start_pfn, unsigned long *end_pfn)
7110{
Tejun Heoc13291a2011-07-12 10:46:30 +02007111 unsigned long this_start_pfn, this_end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07007112 int i;
Tejun Heoc13291a2011-07-12 10:46:30 +02007113
Mel Gormanc7132162006-09-27 01:49:43 -07007114 *start_pfn = -1UL;
7115 *end_pfn = 0;
7116
Tejun Heoc13291a2011-07-12 10:46:30 +02007117 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
7118 *start_pfn = min(*start_pfn, this_start_pfn);
7119 *end_pfn = max(*end_pfn, this_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07007120 }
7121
Christoph Lameter633c0662007-10-16 01:25:37 -07007122 if (*start_pfn == -1UL)
Mel Gormanc7132162006-09-27 01:49:43 -07007123 *start_pfn = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07007124}
7125
7126/*
Mel Gorman2a1e2742007-07-17 04:03:12 -07007127 * This finds a zone that can be used for ZONE_MOVABLE pages. The
7128 * assumption is made that zones within a node are ordered in monotonic
7129 * increasing memory addresses so that the "highest" populated zone is used
7130 */
Adrian Bunkb69a7282008-07-23 21:28:12 -07007131static void __init find_usable_zone_for_movable(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007132{
7133 int zone_index;
7134 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
7135 if (zone_index == ZONE_MOVABLE)
7136 continue;
7137
7138 if (arch_zone_highest_possible_pfn[zone_index] >
7139 arch_zone_lowest_possible_pfn[zone_index])
7140 break;
7141 }
7142
7143 VM_BUG_ON(zone_index == -1);
7144 movable_zone = zone_index;
7145}
7146
7147/*
7148 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
Lucas De Marchi25985ed2011-03-30 22:57:33 -03007149 * because it is sized independent of architecture. Unlike the other zones,
Mel Gorman2a1e2742007-07-17 04:03:12 -07007150 * the starting point for ZONE_MOVABLE is not fixed. It may be different
7151 * in each node depending on the size of each node and how evenly kernelcore
7152 * is distributed. This helper function adjusts the zone ranges
7153 * provided by the architecture for a given node by using the end of the
7154 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
7155 * zones within a node are in order of monotonic increases memory addresses
7156 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08007157static void __init adjust_zone_range_for_zone_movable(int nid,
Mel Gorman2a1e2742007-07-17 04:03:12 -07007158 unsigned long zone_type,
7159 unsigned long node_start_pfn,
7160 unsigned long node_end_pfn,
7161 unsigned long *zone_start_pfn,
7162 unsigned long *zone_end_pfn)
7163{
7164 /* Only adjust if ZONE_MOVABLE is on this node */
7165 if (zone_movable_pfn[nid]) {
7166 /* Size ZONE_MOVABLE */
7167 if (zone_type == ZONE_MOVABLE) {
7168 *zone_start_pfn = zone_movable_pfn[nid];
7169 *zone_end_pfn = min(node_end_pfn,
7170 arch_zone_highest_possible_pfn[movable_zone]);
7171
Xishi Qiue506b992016-10-07 16:58:06 -07007172 /* Adjust for ZONE_MOVABLE starting within this range */
7173 } else if (!mirrored_kernelcore &&
7174 *zone_start_pfn < zone_movable_pfn[nid] &&
7175 *zone_end_pfn > zone_movable_pfn[nid]) {
7176 *zone_end_pfn = zone_movable_pfn[nid];
7177
Mel Gorman2a1e2742007-07-17 04:03:12 -07007178 /* Check if this whole range is within ZONE_MOVABLE */
7179 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
7180 *zone_start_pfn = *zone_end_pfn;
7181 }
7182}
7183
7184/*
Mel Gormanc7132162006-09-27 01:49:43 -07007185 * Return the number of pages a zone spans in a node, including holes
7186 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
7187 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08007188static unsigned long __init zone_spanned_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07007189 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07007190 unsigned long node_start_pfn,
7191 unsigned long node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07007192 unsigned long *zone_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07007193 unsigned long *zone_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07007194{
Linxu Fang299c83d2019-05-13 17:19:17 -07007195 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
7196 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Xishi Qiub5685e92015-09-08 15:04:16 -07007197 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07007198 if (!node_start_pfn && !node_end_pfn)
7199 return 0;
7200
Zhang Yanfei7960aed2013-07-08 15:59:52 -07007201 /* Get the start and end of the zone */
Linxu Fang299c83d2019-05-13 17:19:17 -07007202 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
7203 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007204 adjust_zone_range_for_zone_movable(nid, zone_type,
7205 node_start_pfn, node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07007206 zone_start_pfn, zone_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07007207
7208 /* Check that this node has pages within the zone's required range */
Taku Izumid91749c2016-03-15 14:55:18 -07007209 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07007210 return 0;
7211
7212 /* Move the zone boundaries inside the node if necessary */
Taku Izumid91749c2016-03-15 14:55:18 -07007213 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
7214 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07007215
7216 /* Return the spanned pages */
Taku Izumid91749c2016-03-15 14:55:18 -07007217 return *zone_end_pfn - *zone_start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07007218}
7219
7220/*
7221 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007222 * then all holes in the requested range will be accounted for.
Mel Gormanc7132162006-09-27 01:49:43 -07007223 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08007224unsigned long __init __absent_pages_in_range(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07007225 unsigned long range_start_pfn,
7226 unsigned long range_end_pfn)
7227{
Tejun Heo96e907d2011-07-12 10:46:29 +02007228 unsigned long nr_absent = range_end_pfn - range_start_pfn;
7229 unsigned long start_pfn, end_pfn;
7230 int i;
Mel Gormanc7132162006-09-27 01:49:43 -07007231
Tejun Heo96e907d2011-07-12 10:46:29 +02007232 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
7233 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
7234 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
7235 nr_absent -= end_pfn - start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07007236 }
Tejun Heo96e907d2011-07-12 10:46:29 +02007237 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07007238}
7239
7240/**
7241 * absent_pages_in_range - Return number of page frames in holes within a range
7242 * @start_pfn: The start PFN to start searching for holes
7243 * @end_pfn: The end PFN to stop searching for holes
7244 *
Mike Rapoporta862f682019-03-05 15:48:42 -08007245 * Return: the number of pages frames in memory holes within a range.
Mel Gormanc7132162006-09-27 01:49:43 -07007246 */
7247unsigned long __init absent_pages_in_range(unsigned long start_pfn,
7248 unsigned long end_pfn)
7249{
7250 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
7251}
7252
7253/* Return the number of page frames in holes in a zone on a node */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08007254static unsigned long __init zone_absent_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07007255 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07007256 unsigned long node_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07007257 unsigned long node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07007258{
Tejun Heo96e907d2011-07-12 10:46:29 +02007259 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
7260 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Mel Gorman9c7cd682006-09-27 01:49:58 -07007261 unsigned long zone_start_pfn, zone_end_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07007262 unsigned long nr_absent;
Mel Gorman9c7cd682006-09-27 01:49:58 -07007263
Xishi Qiub5685e92015-09-08 15:04:16 -07007264 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07007265 if (!node_start_pfn && !node_end_pfn)
7266 return 0;
7267
Tejun Heo96e907d2011-07-12 10:46:29 +02007268 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
7269 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman9c7cd682006-09-27 01:49:58 -07007270
Mel Gorman2a1e2742007-07-17 04:03:12 -07007271 adjust_zone_range_for_zone_movable(nid, zone_type,
7272 node_start_pfn, node_end_pfn,
7273 &zone_start_pfn, &zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07007274 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
7275
7276 /*
7277 * ZONE_MOVABLE handling.
7278 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
7279 * and vice versa.
7280 */
Xishi Qiue506b992016-10-07 16:58:06 -07007281 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
7282 unsigned long start_pfn, end_pfn;
7283 struct memblock_region *r;
Taku Izumi342332e2016-03-15 14:55:22 -07007284
Mike Rapoportcc6de162020-10-13 16:58:30 -07007285 for_each_mem_region(r) {
Xishi Qiue506b992016-10-07 16:58:06 -07007286 start_pfn = clamp(memblock_region_memory_base_pfn(r),
7287 zone_start_pfn, zone_end_pfn);
7288 end_pfn = clamp(memblock_region_memory_end_pfn(r),
7289 zone_start_pfn, zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07007290
Xishi Qiue506b992016-10-07 16:58:06 -07007291 if (zone_type == ZONE_MOVABLE &&
7292 memblock_is_mirror(r))
7293 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07007294
Xishi Qiue506b992016-10-07 16:58:06 -07007295 if (zone_type == ZONE_NORMAL &&
7296 !memblock_is_mirror(r))
7297 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07007298 }
7299 }
7300
7301 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07007302}
Mel Gorman0e0b8642006-09-27 01:49:56 -07007303
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08007304static void __init calculate_node_totalpages(struct pglist_data *pgdat,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07007305 unsigned long node_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07007306 unsigned long node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07007307{
Gu Zhengfebd5942015-06-24 16:57:02 -07007308 unsigned long realtotalpages = 0, totalpages = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07007309 enum zone_type i;
7310
Gu Zhengfebd5942015-06-24 16:57:02 -07007311 for (i = 0; i < MAX_NR_ZONES; i++) {
7312 struct zone *zone = pgdat->node_zones + i;
Taku Izumid91749c2016-03-15 14:55:18 -07007313 unsigned long zone_start_pfn, zone_end_pfn;
Mike Rapoport3f08a302020-06-03 15:57:02 -07007314 unsigned long spanned, absent;
Gu Zhengfebd5942015-06-24 16:57:02 -07007315 unsigned long size, real_size;
Mel Gormanc7132162006-09-27 01:49:43 -07007316
Mike Rapoport854e8842020-06-03 15:58:13 -07007317 spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
7318 node_start_pfn,
7319 node_end_pfn,
7320 &zone_start_pfn,
7321 &zone_end_pfn);
7322 absent = zone_absent_pages_in_node(pgdat->node_id, i,
7323 node_start_pfn,
7324 node_end_pfn);
Mike Rapoport3f08a302020-06-03 15:57:02 -07007325
7326 size = spanned;
7327 real_size = size - absent;
7328
Taku Izumid91749c2016-03-15 14:55:18 -07007329 if (size)
7330 zone->zone_start_pfn = zone_start_pfn;
7331 else
7332 zone->zone_start_pfn = 0;
Gu Zhengfebd5942015-06-24 16:57:02 -07007333 zone->spanned_pages = size;
7334 zone->present_pages = real_size;
David Hildenbrand4b097002021-09-07 19:55:19 -07007335#if defined(CONFIG_MEMORY_HOTPLUG)
7336 zone->present_early_pages = real_size;
7337#endif
Gu Zhengfebd5942015-06-24 16:57:02 -07007338
7339 totalpages += size;
7340 realtotalpages += real_size;
7341 }
7342
7343 pgdat->node_spanned_pages = totalpages;
Mel Gormanc7132162006-09-27 01:49:43 -07007344 pgdat->node_present_pages = realtotalpages;
Heiner Kallweit9660eca2021-06-28 19:41:31 -07007345 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
Mel Gormanc7132162006-09-27 01:49:43 -07007346}
7347
Mel Gorman835c1342007-10-16 01:25:47 -07007348#ifndef CONFIG_SPARSEMEM
7349/*
7350 * Calculate the size of the zone->blockflags rounded to an unsigned long
Mel Gormand9c23402007-10-16 01:26:01 -07007351 * Start by making sure zonesize is a multiple of pageblock_order by rounding
7352 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
Mel Gorman835c1342007-10-16 01:25:47 -07007353 * round what is now in bits to nearest long in bits, then return it in
7354 * bytes.
7355 */
Linus Torvalds7c455122013-02-18 09:58:02 -08007356static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
Mel Gorman835c1342007-10-16 01:25:47 -07007357{
7358 unsigned long usemapsize;
7359
Linus Torvalds7c455122013-02-18 09:58:02 -08007360 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
Mel Gormand9c23402007-10-16 01:26:01 -07007361 usemapsize = roundup(zonesize, pageblock_nr_pages);
7362 usemapsize = usemapsize >> pageblock_order;
Mel Gorman835c1342007-10-16 01:25:47 -07007363 usemapsize *= NR_PAGEBLOCK_BITS;
7364 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
7365
7366 return usemapsize / 8;
7367}
7368
Baoquan He7010a6e2021-02-24 12:06:20 -08007369static void __ref setup_usemap(struct zone *zone)
Mel Gorman835c1342007-10-16 01:25:47 -07007370{
Baoquan He7010a6e2021-02-24 12:06:20 -08007371 unsigned long usemapsize = usemap_size(zone->zone_start_pfn,
7372 zone->spanned_pages);
Mel Gorman835c1342007-10-16 01:25:47 -07007373 zone->pageblock_flags = NULL;
Mike Rapoport23a70522019-03-05 15:46:43 -08007374 if (usemapsize) {
Santosh Shilimkar67828322014-01-21 15:50:25 -08007375 zone->pageblock_flags =
Mike Rapoport26fb3da2019-03-11 23:30:42 -07007376 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
Baoquan He7010a6e2021-02-24 12:06:20 -08007377 zone_to_nid(zone));
Mike Rapoport23a70522019-03-05 15:46:43 -08007378 if (!zone->pageblock_flags)
7379 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
Baoquan He7010a6e2021-02-24 12:06:20 -08007380 usemapsize, zone->name, zone_to_nid(zone));
Mike Rapoport23a70522019-03-05 15:46:43 -08007381 }
Mel Gorman835c1342007-10-16 01:25:47 -07007382}
7383#else
Baoquan He7010a6e2021-02-24 12:06:20 -08007384static inline void setup_usemap(struct zone *zone) {}
Mel Gorman835c1342007-10-16 01:25:47 -07007385#endif /* CONFIG_SPARSEMEM */
7386
Mel Gormand9c23402007-10-16 01:26:01 -07007387#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Mel Gormanba72cb82007-11-28 16:21:13 -08007388
Mel Gormand9c23402007-10-16 01:26:01 -07007389/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
Oscar Salvador03e85f92018-08-21 21:53:43 -07007390void __init set_pageblock_order(void)
Mel Gormand9c23402007-10-16 01:26:01 -07007391{
Andrew Morton955c1cd2012-05-29 15:06:31 -07007392 unsigned int order;
7393
Mel Gormand9c23402007-10-16 01:26:01 -07007394 /* Check that pageblock_nr_pages has not already been setup */
7395 if (pageblock_order)
7396 return;
7397
Andrew Morton955c1cd2012-05-29 15:06:31 -07007398 if (HPAGE_SHIFT > PAGE_SHIFT)
7399 order = HUGETLB_PAGE_ORDER;
7400 else
7401 order = MAX_ORDER - 1;
7402
Mel Gormand9c23402007-10-16 01:26:01 -07007403 /*
7404 * Assume the largest contiguous order of interest is a huge page.
Andrew Morton955c1cd2012-05-29 15:06:31 -07007405 * This value may be variable depending on boot parameters on IA64 and
7406 * powerpc.
Mel Gormand9c23402007-10-16 01:26:01 -07007407 */
7408 pageblock_order = order;
7409}
7410#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
7411
Mel Gormanba72cb82007-11-28 16:21:13 -08007412/*
7413 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
Andrew Morton955c1cd2012-05-29 15:06:31 -07007414 * is unused as pageblock_order is set at compile-time. See
7415 * include/linux/pageblock-flags.h for the values of pageblock_order based on
7416 * the kernel config
Mel Gormanba72cb82007-11-28 16:21:13 -08007417 */
Oscar Salvador03e85f92018-08-21 21:53:43 -07007418void __init set_pageblock_order(void)
Mel Gormanba72cb82007-11-28 16:21:13 -08007419{
Mel Gormanba72cb82007-11-28 16:21:13 -08007420}
Mel Gormand9c23402007-10-16 01:26:01 -07007421
7422#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
7423
Oscar Salvador03e85f92018-08-21 21:53:43 -07007424static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
Pavel Tatashin7cc2a952018-08-21 21:53:36 -07007425 unsigned long present_pages)
Jiang Liu01cefae2012-12-12 13:52:19 -08007426{
7427 unsigned long pages = spanned_pages;
7428
7429 /*
7430 * Provide a more accurate estimation if there are holes within
7431 * the zone and SPARSEMEM is in use. If there are holes within the
7432 * zone, each populated memory region may cost us one or two extra
7433 * memmap pages due to alignment because memmap pages for each
Masahiro Yamada89d790a2017-02-27 14:29:01 -08007434 * populated regions may not be naturally aligned on page boundary.
Jiang Liu01cefae2012-12-12 13:52:19 -08007435 * So the (present_pages >> 4) heuristic is a tradeoff for that.
7436 */
7437 if (spanned_pages > present_pages + (present_pages >> 4) &&
7438 IS_ENABLED(CONFIG_SPARSEMEM))
7439 pages = present_pages;
7440
7441 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
7442}
7443
Oscar Salvadorace1db32018-08-21 21:53:29 -07007444#ifdef CONFIG_TRANSPARENT_HUGEPAGE
7445static void pgdat_init_split_queue(struct pglist_data *pgdat)
7446{
Yang Shi364c1ee2019-09-23 15:38:06 -07007447 struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
7448
7449 spin_lock_init(&ds_queue->split_queue_lock);
7450 INIT_LIST_HEAD(&ds_queue->split_queue);
7451 ds_queue->split_queue_len = 0;
Oscar Salvadorace1db32018-08-21 21:53:29 -07007452}
7453#else
7454static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
7455#endif
7456
7457#ifdef CONFIG_COMPACTION
7458static void pgdat_init_kcompactd(struct pglist_data *pgdat)
7459{
7460 init_waitqueue_head(&pgdat->kcompactd_wait);
7461}
7462#else
7463static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
7464#endif
7465
Oscar Salvador03e85f92018-08-21 21:53:43 -07007466static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007467{
Mel Gorman8cd7c582021-11-05 13:42:25 -07007468 int i;
7469
Dave Hansen208d54e2005-10-29 18:16:52 -07007470 pgdat_resize_init(pgdat);
Oscar Salvadorace1db32018-08-21 21:53:29 -07007471
Oscar Salvadorace1db32018-08-21 21:53:29 -07007472 pgdat_init_split_queue(pgdat);
7473 pgdat_init_kcompactd(pgdat);
7474
Linus Torvalds1da177e2005-04-16 15:20:36 -07007475 init_waitqueue_head(&pgdat->kswapd_wait);
Mel Gorman55150612012-07-31 16:44:35 -07007476 init_waitqueue_head(&pgdat->pfmemalloc_wait);
Oscar Salvadorace1db32018-08-21 21:53:29 -07007477
Mel Gorman8cd7c582021-11-05 13:42:25 -07007478 for (i = 0; i < NR_VMSCAN_THROTTLE; i++)
7479 init_waitqueue_head(&pgdat->reclaim_wait[i]);
7480
Joonsoo Kimeefa864b2014-12-12 16:55:46 -08007481 pgdat_page_ext_init(pgdat);
Johannes Weiner867e5e12019-11-30 17:55:34 -08007482 lruvec_init(&pgdat->__lruvec);
Oscar Salvador03e85f92018-08-21 21:53:43 -07007483}
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01007484
Oscar Salvador03e85f92018-08-21 21:53:43 -07007485static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
7486 unsigned long remaining_pages)
7487{
Arun KS9705bea2018-12-28 00:34:24 -08007488 atomic_long_set(&zone->managed_pages, remaining_pages);
Oscar Salvador03e85f92018-08-21 21:53:43 -07007489 zone_set_nid(zone, nid);
7490 zone->name = zone_names[idx];
7491 zone->zone_pgdat = NODE_DATA(nid);
7492 spin_lock_init(&zone->lock);
7493 zone_seqlock_init(zone);
7494 zone_pcp_init(zone);
7495}
7496
7497/*
7498 * Set up the zone data structures
7499 * - init pgdat internals
7500 * - init all zones belonging to this node
7501 *
7502 * NOTE: this function is only called during memory hotplug
7503 */
7504#ifdef CONFIG_MEMORY_HOTPLUG
7505void __ref free_area_init_core_hotplug(int nid)
7506{
7507 enum zone_type z;
7508 pg_data_t *pgdat = NODE_DATA(nid);
7509
7510 pgdat_init_internals(pgdat);
7511 for (z = 0; z < MAX_NR_ZONES; z++)
7512 zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
7513}
7514#endif
7515
7516/*
7517 * Set up the zone data structures:
7518 * - mark all pages reserved
7519 * - mark all memory queues empty
7520 * - clear the memory bitmaps
7521 *
7522 * NOTE: pgdat should get zeroed by caller.
7523 * NOTE: this function is only called during early init.
7524 */
7525static void __init free_area_init_core(struct pglist_data *pgdat)
7526{
7527 enum zone_type j;
7528 int nid = pgdat->node_id;
7529
7530 pgdat_init_internals(pgdat);
Johannes Weiner385386c2017-07-06 15:40:43 -07007531 pgdat->per_cpu_nodestats = &boot_nodestats;
7532
Linus Torvalds1da177e2005-04-16 15:20:36 -07007533 for (j = 0; j < MAX_NR_ZONES; j++) {
7534 struct zone *zone = pgdat->node_zones + j;
Wei Yange6943852018-06-07 17:06:04 -07007535 unsigned long size, freesize, memmap_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007536
Gu Zhengfebd5942015-06-24 16:57:02 -07007537 size = zone->spanned_pages;
Wei Yange6943852018-06-07 17:06:04 -07007538 freesize = zone->present_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007539
Mel Gorman0e0b8642006-09-27 01:49:56 -07007540 /*
Jiang Liu9feedc92012-12-12 13:52:12 -08007541 * Adjust freesize so that it accounts for how much memory
Mel Gorman0e0b8642006-09-27 01:49:56 -07007542 * is used by this zone for memmap. This affects the watermark
7543 * and per-cpu initialisations
7544 */
Wei Yange6943852018-06-07 17:06:04 -07007545 memmap_pages = calc_memmap_size(size, freesize);
Zhong Hongboba914f42014-12-12 16:56:21 -08007546 if (!is_highmem_idx(j)) {
7547 if (freesize >= memmap_pages) {
7548 freesize -= memmap_pages;
7549 if (memmap_pages)
Heiner Kallweit9660eca2021-06-28 19:41:31 -07007550 pr_debug(" %s zone: %lu pages used for memmap\n",
7551 zone_names[j], memmap_pages);
Zhong Hongboba914f42014-12-12 16:56:21 -08007552 } else
Dong Aishenge47aa902021-06-28 19:42:30 -07007553 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n",
Zhong Hongboba914f42014-12-12 16:56:21 -08007554 zone_names[j], memmap_pages, freesize);
7555 }
Mel Gorman0e0b8642006-09-27 01:49:56 -07007556
Christoph Lameter62672762007-02-10 01:43:07 -08007557 /* Account for reserved pages */
Jiang Liu9feedc92012-12-12 13:52:12 -08007558 if (j == 0 && freesize > dma_reserve) {
7559 freesize -= dma_reserve;
Heiner Kallweit9660eca2021-06-28 19:41:31 -07007560 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve);
Mel Gorman0e0b8642006-09-27 01:49:56 -07007561 }
7562
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07007563 if (!is_highmem_idx(j))
Jiang Liu9feedc92012-12-12 13:52:12 -08007564 nr_kernel_pages += freesize;
Jiang Liu01cefae2012-12-12 13:52:19 -08007565 /* Charge for highmem memmap if there are enough kernel pages */
7566 else if (nr_kernel_pages > memmap_pages * 2)
7567 nr_kernel_pages -= memmap_pages;
Jiang Liu9feedc92012-12-12 13:52:12 -08007568 nr_all_pages += freesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007569
Jiang Liu9feedc92012-12-12 13:52:12 -08007570 /*
7571 * Set an approximate value for lowmem here, it will be adjusted
7572 * when the bootmem allocator frees pages into the buddy system.
7573 * And all highmem pages will be managed by the buddy system.
7574 */
Oscar Salvador03e85f92018-08-21 21:53:43 -07007575 zone_init_internals(zone, j, nid, freesize);
Johannes Weiner81c0a2b2013-09-11 14:20:47 -07007576
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09007577 if (!size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007578 continue;
7579
Andrew Morton955c1cd2012-05-29 15:06:31 -07007580 set_pageblock_order();
Baoquan He7010a6e2021-02-24 12:06:20 -08007581 setup_usemap(zone);
Baoquan He9699ee72021-02-24 12:06:24 -08007582 init_currently_empty_zone(zone, zone->zone_start_pfn, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007583 }
7584}
7585
Mike Rapoport43b02ba2021-06-28 19:43:05 -07007586#ifdef CONFIG_FLATMEM
Mike Rapoport3b446da2021-09-02 14:58:10 -07007587static void __init alloc_node_mem_map(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007588{
Tony Luckb0aeba72015-11-10 10:09:47 -08007589 unsigned long __maybe_unused start = 0;
Laura Abbotta1c34a32015-11-05 18:48:46 -08007590 unsigned long __maybe_unused offset = 0;
7591
Linus Torvalds1da177e2005-04-16 15:20:36 -07007592 /* Skip empty nodes */
7593 if (!pgdat->node_spanned_pages)
7594 return;
7595
Tony Luckb0aeba72015-11-10 10:09:47 -08007596 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
7597 offset = pgdat->node_start_pfn - start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007598 /* ia64 gets its own node_mem_map, before this, without bootmem */
7599 if (!pgdat->node_mem_map) {
Tony Luckb0aeba72015-11-10 10:09:47 -08007600 unsigned long size, end;
Andy Whitcroftd41dee32005-06-23 00:07:54 -07007601 struct page *map;
7602
Bob Piccoe984bb42006-05-20 15:00:31 -07007603 /*
7604 * The zone's endpoints aren't required to be MAX_ORDER
7605 * aligned but the node_mem_map endpoints must be in order
7606 * for the buddy allocator to function correctly.
7607 */
Cody P Schafer108bcc92013-02-22 16:35:23 -08007608 end = pgdat_end_pfn(pgdat);
Bob Piccoe984bb42006-05-20 15:00:31 -07007609 end = ALIGN(end, MAX_ORDER_NR_PAGES);
7610 size = (end - start) * sizeof(struct page);
Mike Rapoportc803b3c2021-09-02 14:58:02 -07007611 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT,
7612 pgdat->node_id, false);
Mike Rapoport23a70522019-03-05 15:46:43 -08007613 if (!map)
7614 panic("Failed to allocate %ld bytes for node %d memory map\n",
7615 size, pgdat->node_id);
Laura Abbotta1c34a32015-11-05 18:48:46 -08007616 pgdat->node_mem_map = map + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007617 }
Oscar Salvador0cd842f2017-11-15 17:39:18 -08007618 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
7619 __func__, pgdat->node_id, (unsigned long)pgdat,
7620 (unsigned long)pgdat->node_mem_map);
Mike Rapoporta9ee6cf2021-06-28 19:43:01 -07007621#ifndef CONFIG_NUMA
Linus Torvalds1da177e2005-04-16 15:20:36 -07007622 /*
7623 * With no DISCONTIG, the global mem_map is just set as node 0's
7624 */
Mel Gormanc7132162006-09-27 01:49:43 -07007625 if (pgdat == NODE_DATA(0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007626 mem_map = NODE_DATA(0)->node_mem_map;
Mel Gormanc7132162006-09-27 01:49:43 -07007627 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
Laura Abbotta1c34a32015-11-05 18:48:46 -08007628 mem_map -= offset;
Mel Gormanc7132162006-09-27 01:49:43 -07007629 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007630#endif
7631}
Oscar Salvador0cd842f2017-11-15 17:39:18 -08007632#else
Mike Rapoport3b446da2021-09-02 14:58:10 -07007633static inline void alloc_node_mem_map(struct pglist_data *pgdat) { }
Mike Rapoport43b02ba2021-06-28 19:43:05 -07007634#endif /* CONFIG_FLATMEM */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007635
Oscar Salvador0188dc92018-08-21 21:53:39 -07007636#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
7637static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
7638{
Oscar Salvador0188dc92018-08-21 21:53:39 -07007639 pgdat->first_deferred_pfn = ULONG_MAX;
7640}
7641#else
7642static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
7643#endif
7644
Mike Rapoport854e8842020-06-03 15:58:13 -07007645static void __init free_area_init_node(int nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007646{
Johannes Weiner9109fb72008-07-23 21:27:20 -07007647 pg_data_t *pgdat = NODE_DATA(nid);
Zhang Yanfei7960aed2013-07-08 15:59:52 -07007648 unsigned long start_pfn = 0;
7649 unsigned long end_pfn = 0;
Johannes Weiner9109fb72008-07-23 21:27:20 -07007650
Minchan Kim88fdf752012-07-31 16:46:14 -07007651 /* pg_data_t should be reset to zero when it's allocated */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07007652 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
Minchan Kim88fdf752012-07-31 16:46:14 -07007653
Mike Rapoport854e8842020-06-03 15:58:13 -07007654 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07007655
Linus Torvalds1da177e2005-04-16 15:20:36 -07007656 pgdat->node_id = nid;
Mike Rapoport854e8842020-06-03 15:58:13 -07007657 pgdat->node_start_pfn = start_pfn;
Mel Gorman75ef7182016-07-28 15:45:24 -07007658 pgdat->per_cpu_nodestats = NULL;
Mike Rapoport854e8842020-06-03 15:58:13 -07007659
Juergen Gross8d29e182015-02-11 15:26:01 -08007660 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
Zhen Lei4ada0c52015-09-08 15:04:19 -07007661 (u64)start_pfn << PAGE_SHIFT,
7662 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
Mike Rapoport854e8842020-06-03 15:58:13 -07007663 calculate_node_totalpages(pgdat, start_pfn, end_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007664
7665 alloc_node_mem_map(pgdat);
Oscar Salvador0188dc92018-08-21 21:53:39 -07007666 pgdat_set_deferred_range(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007667
Wei Yang7f3eb552015-09-08 14:59:50 -07007668 free_area_init_core(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007669}
7670
Mike Rapoportbc9331a2020-06-03 15:58:09 -07007671void __init free_area_init_memoryless_node(int nid)
Mike Rapoport3f08a302020-06-03 15:57:02 -07007672{
Mike Rapoport854e8842020-06-03 15:58:13 -07007673 free_area_init_node(nid);
Mike Rapoport3f08a302020-06-03 15:57:02 -07007674}
7675
Miklos Szeredi418508c2007-05-23 13:57:55 -07007676#if MAX_NUMNODES > 1
7677/*
7678 * Figure out the number of possible node ids.
7679 */
Cody P Schaferf9872ca2013-04-29 15:08:01 -07007680void __init setup_nr_node_ids(void)
Miklos Szeredi418508c2007-05-23 13:57:55 -07007681{
Wei Yang904a9552015-09-08 14:59:48 -07007682 unsigned int highest;
Miklos Szeredi418508c2007-05-23 13:57:55 -07007683
Wei Yang904a9552015-09-08 14:59:48 -07007684 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
Miklos Szeredi418508c2007-05-23 13:57:55 -07007685 nr_node_ids = highest + 1;
7686}
Miklos Szeredi418508c2007-05-23 13:57:55 -07007687#endif
7688
Mel Gormanc7132162006-09-27 01:49:43 -07007689/**
Tejun Heo1e019792011-07-12 09:45:34 +02007690 * node_map_pfn_alignment - determine the maximum internode alignment
7691 *
7692 * This function should be called after node map is populated and sorted.
7693 * It calculates the maximum power of two alignment which can distinguish
7694 * all the nodes.
7695 *
7696 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
7697 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
7698 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
7699 * shifted, 1GiB is enough and this function will indicate so.
7700 *
7701 * This is used to test whether pfn -> nid mapping of the chosen memory
7702 * model has fine enough granularity to avoid incorrect mapping for the
7703 * populated node map.
7704 *
Mike Rapoporta862f682019-03-05 15:48:42 -08007705 * Return: the determined alignment in pfn's. 0 if there is no alignment
Tejun Heo1e019792011-07-12 09:45:34 +02007706 * requirement (single node).
7707 */
7708unsigned long __init node_map_pfn_alignment(void)
7709{
7710 unsigned long accl_mask = 0, last_end = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02007711 unsigned long start, end, mask;
Anshuman Khandual98fa15f2019-03-05 15:42:58 -08007712 int last_nid = NUMA_NO_NODE;
Tejun Heoc13291a2011-07-12 10:46:30 +02007713 int i, nid;
Tejun Heo1e019792011-07-12 09:45:34 +02007714
Tejun Heoc13291a2011-07-12 10:46:30 +02007715 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
Tejun Heo1e019792011-07-12 09:45:34 +02007716 if (!start || last_nid < 0 || last_nid == nid) {
7717 last_nid = nid;
7718 last_end = end;
7719 continue;
7720 }
7721
7722 /*
7723 * Start with a mask granular enough to pin-point to the
7724 * start pfn and tick off bits one-by-one until it becomes
7725 * too coarse to separate the current node from the last.
7726 */
7727 mask = ~((1 << __ffs(start)) - 1);
7728 while (mask && last_end <= (start & (mask << 1)))
7729 mask <<= 1;
7730
7731 /* accumulate all internode masks */
7732 accl_mask |= mask;
7733 }
7734
7735 /* convert mask to number of pages */
7736 return ~accl_mask + 1;
7737}
7738
Mel Gormanc7132162006-09-27 01:49:43 -07007739/**
7740 * find_min_pfn_with_active_regions - Find the minimum PFN registered
7741 *
Mike Rapoporta862f682019-03-05 15:48:42 -08007742 * Return: the minimum PFN based on information provided via
Zhang Zhen7d018172014-06-04 16:10:53 -07007743 * memblock_set_node().
Mel Gormanc7132162006-09-27 01:49:43 -07007744 */
7745unsigned long __init find_min_pfn_with_active_regions(void)
7746{
Mike Rapoport8a1b25f2020-06-03 15:58:18 -07007747 return PHYS_PFN(memblock_start_of_DRAM());
Mel Gormanc7132162006-09-27 01:49:43 -07007748}
7749
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007750/*
7751 * early_calculate_totalpages()
7752 * Sum pages in active regions for movable zone.
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007753 * Populate N_MEMORY for calculating usable_nodes.
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007754 */
Adrian Bunk484f51f2007-10-16 01:26:03 -07007755static unsigned long __init early_calculate_totalpages(void)
Mel Gorman7e63efef2007-07-17 04:03:15 -07007756{
Mel Gorman7e63efef2007-07-17 04:03:15 -07007757 unsigned long totalpages = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02007758 unsigned long start_pfn, end_pfn;
7759 int i, nid;
Mel Gorman7e63efef2007-07-17 04:03:15 -07007760
Tejun Heoc13291a2011-07-12 10:46:30 +02007761 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
7762 unsigned long pages = end_pfn - start_pfn;
7763
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007764 totalpages += pages;
7765 if (pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007766 node_set_state(nid, N_MEMORY);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007767 }
Pintu Kumarb8af2942013-09-11 14:20:34 -07007768 return totalpages;
Mel Gorman7e63efef2007-07-17 04:03:15 -07007769}
7770
Mel Gorman2a1e2742007-07-17 04:03:12 -07007771/*
7772 * Find the PFN the Movable zone begins in each node. Kernel memory
7773 * is spread evenly between nodes as long as the nodes have enough
7774 * memory. When they don't, some nodes will have more kernelcore than
7775 * others
7776 */
Kautuk Consulb224ef82012-03-21 16:34:15 -07007777static void __init find_zone_movable_pfns_for_nodes(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007778{
7779 int i, nid;
7780 unsigned long usable_startpfn;
7781 unsigned long kernelcore_node, kernelcore_remaining;
Yinghai Lu66918dc2009-06-30 11:41:37 -07007782 /* save the state before borrow the nodemask */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007783 nodemask_t saved_node_state = node_states[N_MEMORY];
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007784 unsigned long totalpages = early_calculate_totalpages();
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007785 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
Emil Medve136199f2014-04-07 15:37:52 -07007786 struct memblock_region *r;
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007787
7788 /* Need to find movable_zone earlier when movable_node is specified. */
7789 find_usable_zone_for_movable();
Mel Gorman2a1e2742007-07-17 04:03:12 -07007790
Mel Gorman7e63efef2007-07-17 04:03:15 -07007791 /*
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007792 * If movable_node is specified, ignore kernelcore and movablecore
7793 * options.
7794 */
7795 if (movable_node_is_enabled()) {
Mike Rapoportcc6de162020-10-13 16:58:30 -07007796 for_each_mem_region(r) {
Emil Medve136199f2014-04-07 15:37:52 -07007797 if (!memblock_is_hotpluggable(r))
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007798 continue;
7799
Mike Rapoportd622abf2020-06-03 15:56:53 -07007800 nid = memblock_get_region_node(r);
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007801
Emil Medve136199f2014-04-07 15:37:52 -07007802 usable_startpfn = PFN_DOWN(r->base);
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007803 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7804 min(usable_startpfn, zone_movable_pfn[nid]) :
7805 usable_startpfn;
7806 }
7807
7808 goto out2;
7809 }
7810
7811 /*
Taku Izumi342332e2016-03-15 14:55:22 -07007812 * If kernelcore=mirror is specified, ignore movablecore option
7813 */
7814 if (mirrored_kernelcore) {
7815 bool mem_below_4gb_not_mirrored = false;
7816
Mike Rapoportcc6de162020-10-13 16:58:30 -07007817 for_each_mem_region(r) {
Taku Izumi342332e2016-03-15 14:55:22 -07007818 if (memblock_is_mirror(r))
7819 continue;
7820
Mike Rapoportd622abf2020-06-03 15:56:53 -07007821 nid = memblock_get_region_node(r);
Taku Izumi342332e2016-03-15 14:55:22 -07007822
7823 usable_startpfn = memblock_region_memory_base_pfn(r);
7824
7825 if (usable_startpfn < 0x100000) {
7826 mem_below_4gb_not_mirrored = true;
7827 continue;
7828 }
7829
7830 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7831 min(usable_startpfn, zone_movable_pfn[nid]) :
7832 usable_startpfn;
7833 }
7834
7835 if (mem_below_4gb_not_mirrored)
Chen Tao633bf2f2020-06-03 16:00:02 -07007836 pr_warn("This configuration results in unmirrored kernel memory.\n");
Taku Izumi342332e2016-03-15 14:55:22 -07007837
7838 goto out2;
7839 }
7840
7841 /*
David Rientjesa5c6d652018-04-05 16:23:09 -07007842 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
7843 * amount of necessary memory.
7844 */
7845 if (required_kernelcore_percent)
7846 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
7847 10000UL;
7848 if (required_movablecore_percent)
7849 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
7850 10000UL;
7851
7852 /*
7853 * If movablecore= was specified, calculate what size of
Mel Gorman7e63efef2007-07-17 04:03:15 -07007854 * kernelcore that corresponds so that memory usable for
7855 * any allocation type is evenly spread. If both kernelcore
7856 * and movablecore are specified, then the value of kernelcore
7857 * will be used for required_kernelcore if it's greater than
7858 * what movablecore would have allowed.
7859 */
7860 if (required_movablecore) {
Mel Gorman7e63efef2007-07-17 04:03:15 -07007861 unsigned long corepages;
7862
7863 /*
7864 * Round-up so that ZONE_MOVABLE is at least as large as what
7865 * was requested by the user
7866 */
7867 required_movablecore =
7868 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
Xishi Qiu9fd745d2015-11-05 18:48:11 -08007869 required_movablecore = min(totalpages, required_movablecore);
Mel Gorman7e63efef2007-07-17 04:03:15 -07007870 corepages = totalpages - required_movablecore;
7871
7872 required_kernelcore = max(required_kernelcore, corepages);
7873 }
7874
Xishi Qiubde304b2015-11-05 18:48:56 -08007875 /*
7876 * If kernelcore was not specified or kernelcore size is larger
7877 * than totalpages, there is no ZONE_MOVABLE.
7878 */
7879 if (!required_kernelcore || required_kernelcore >= totalpages)
Yinghai Lu66918dc2009-06-30 11:41:37 -07007880 goto out;
Mel Gorman2a1e2742007-07-17 04:03:12 -07007881
7882 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
Mel Gorman2a1e2742007-07-17 04:03:12 -07007883 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
7884
7885restart:
7886 /* Spread kernelcore memory as evenly as possible throughout nodes */
7887 kernelcore_node = required_kernelcore / usable_nodes;
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007888 for_each_node_state(nid, N_MEMORY) {
Tejun Heoc13291a2011-07-12 10:46:30 +02007889 unsigned long start_pfn, end_pfn;
7890
Mel Gorman2a1e2742007-07-17 04:03:12 -07007891 /*
7892 * Recalculate kernelcore_node if the division per node
7893 * now exceeds what is necessary to satisfy the requested
7894 * amount of memory for the kernel
7895 */
7896 if (required_kernelcore < kernelcore_node)
7897 kernelcore_node = required_kernelcore / usable_nodes;
7898
7899 /*
7900 * As the map is walked, we track how much memory is usable
7901 * by the kernel using kernelcore_remaining. When it is
7902 * 0, the rest of the node is usable by ZONE_MOVABLE
7903 */
7904 kernelcore_remaining = kernelcore_node;
7905
7906 /* Go through each range of PFNs within this node */
Tejun Heoc13291a2011-07-12 10:46:30 +02007907 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
Mel Gorman2a1e2742007-07-17 04:03:12 -07007908 unsigned long size_pages;
7909
Tejun Heoc13291a2011-07-12 10:46:30 +02007910 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007911 if (start_pfn >= end_pfn)
7912 continue;
7913
7914 /* Account for what is only usable for kernelcore */
7915 if (start_pfn < usable_startpfn) {
7916 unsigned long kernel_pages;
7917 kernel_pages = min(end_pfn, usable_startpfn)
7918 - start_pfn;
7919
7920 kernelcore_remaining -= min(kernel_pages,
7921 kernelcore_remaining);
7922 required_kernelcore -= min(kernel_pages,
7923 required_kernelcore);
7924
7925 /* Continue if range is now fully accounted */
7926 if (end_pfn <= usable_startpfn) {
7927
7928 /*
7929 * Push zone_movable_pfn to the end so
7930 * that if we have to rebalance
7931 * kernelcore across nodes, we will
7932 * not double account here
7933 */
7934 zone_movable_pfn[nid] = end_pfn;
7935 continue;
7936 }
7937 start_pfn = usable_startpfn;
7938 }
7939
7940 /*
7941 * The usable PFN range for ZONE_MOVABLE is from
7942 * start_pfn->end_pfn. Calculate size_pages as the
7943 * number of pages used as kernelcore
7944 */
7945 size_pages = end_pfn - start_pfn;
7946 if (size_pages > kernelcore_remaining)
7947 size_pages = kernelcore_remaining;
7948 zone_movable_pfn[nid] = start_pfn + size_pages;
7949
7950 /*
7951 * Some kernelcore has been met, update counts and
7952 * break if the kernelcore for this node has been
Pintu Kumarb8af2942013-09-11 14:20:34 -07007953 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07007954 */
7955 required_kernelcore -= min(required_kernelcore,
7956 size_pages);
7957 kernelcore_remaining -= size_pages;
7958 if (!kernelcore_remaining)
7959 break;
7960 }
7961 }
7962
7963 /*
7964 * If there is still required_kernelcore, we do another pass with one
7965 * less node in the count. This will push zone_movable_pfn[nid] further
7966 * along on the nodes that still have memory until kernelcore is
Pintu Kumarb8af2942013-09-11 14:20:34 -07007967 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07007968 */
7969 usable_nodes--;
7970 if (usable_nodes && required_kernelcore > usable_nodes)
7971 goto restart;
7972
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007973out2:
Mel Gorman2a1e2742007-07-17 04:03:12 -07007974 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
7975 for (nid = 0; nid < MAX_NUMNODES; nid++)
7976 zone_movable_pfn[nid] =
7977 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
Yinghai Lu66918dc2009-06-30 11:41:37 -07007978
Yinghai Lu20e69262013-03-01 14:51:27 -08007979out:
Yinghai Lu66918dc2009-06-30 11:41:37 -07007980 /* restore the node_state */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007981 node_states[N_MEMORY] = saved_node_state;
Mel Gorman2a1e2742007-07-17 04:03:12 -07007982}
7983
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007984/* Any regular or high memory on that node ? */
7985static void check_for_memory(pg_data_t *pgdat, int nid)
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007986{
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007987 enum zone_type zone_type;
7988
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007989 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007990 struct zone *zone = &pgdat->node_zones[zone_type];
Xishi Qiub38a8722013-11-12 15:07:20 -08007991 if (populated_zone(zone)) {
Oscar Salvador7b0e0c02018-10-26 15:03:58 -07007992 if (IS_ENABLED(CONFIG_HIGHMEM))
7993 node_set_state(nid, N_HIGH_MEMORY);
7994 if (zone_type <= ZONE_NORMAL)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007995 node_set_state(nid, N_NORMAL_MEMORY);
Bob Liud0048b02012-01-12 17:19:07 -08007996 break;
7997 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007998 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007999}
8000
Mike Rapoport51930df2020-06-03 15:58:03 -07008001/*
Ingo Molnarf0953a12021-05-06 18:06:47 -07008002 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
Mike Rapoport51930df2020-06-03 15:58:03 -07008003 * such cases we allow max_zone_pfn sorted in the descending order
8004 */
8005bool __weak arch_has_descending_max_zone_pfns(void)
8006{
8007 return false;
8008}
8009
Mel Gormanc7132162006-09-27 01:49:43 -07008010/**
Mike Rapoport9691a072020-06-03 15:57:10 -07008011 * free_area_init - Initialise all pg_data_t and zone data
Randy Dunlap88ca3b92006-10-04 02:15:25 -07008012 * @max_zone_pfn: an array of max PFNs for each zone
Mel Gormanc7132162006-09-27 01:49:43 -07008013 *
8014 * This will call free_area_init_node() for each active node in the system.
Zhang Zhen7d018172014-06-04 16:10:53 -07008015 * Using the page ranges provided by memblock_set_node(), the size of each
Mel Gormanc7132162006-09-27 01:49:43 -07008016 * zone in each node and their holes is calculated. If the maximum PFN
8017 * between two adjacent zones match, it is assumed that the zone is empty.
8018 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
8019 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
8020 * starts where the previous one ended. For example, ZONE_DMA32 starts
8021 * at arch_max_dma_pfn.
8022 */
Mike Rapoport9691a072020-06-03 15:57:10 -07008023void __init free_area_init(unsigned long *max_zone_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07008024{
Tejun Heoc13291a2011-07-12 10:46:30 +02008025 unsigned long start_pfn, end_pfn;
Mike Rapoport51930df2020-06-03 15:58:03 -07008026 int i, nid, zone;
8027 bool descending;
Mel Gormana6af2bc2007-02-10 01:42:57 -08008028
Mel Gormanc7132162006-09-27 01:49:43 -07008029 /* Record where the zone boundaries are */
8030 memset(arch_zone_lowest_possible_pfn, 0,
8031 sizeof(arch_zone_lowest_possible_pfn));
8032 memset(arch_zone_highest_possible_pfn, 0,
8033 sizeof(arch_zone_highest_possible_pfn));
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07008034
8035 start_pfn = find_min_pfn_with_active_regions();
Mike Rapoport51930df2020-06-03 15:58:03 -07008036 descending = arch_has_descending_max_zone_pfns();
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07008037
8038 for (i = 0; i < MAX_NR_ZONES; i++) {
Mike Rapoport51930df2020-06-03 15:58:03 -07008039 if (descending)
8040 zone = MAX_NR_ZONES - i - 1;
8041 else
8042 zone = i;
8043
8044 if (zone == ZONE_MOVABLE)
Mel Gorman2a1e2742007-07-17 04:03:12 -07008045 continue;
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07008046
Mike Rapoport51930df2020-06-03 15:58:03 -07008047 end_pfn = max(max_zone_pfn[zone], start_pfn);
8048 arch_zone_lowest_possible_pfn[zone] = start_pfn;
8049 arch_zone_highest_possible_pfn[zone] = end_pfn;
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07008050
8051 start_pfn = end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07008052 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07008053
8054 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
8055 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
Kautuk Consulb224ef82012-03-21 16:34:15 -07008056 find_zone_movable_pfns_for_nodes();
Mel Gormanc7132162006-09-27 01:49:43 -07008057
Mel Gormanc7132162006-09-27 01:49:43 -07008058 /* Print out the zone ranges */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08008059 pr_info("Zone ranges:\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07008060 for (i = 0; i < MAX_NR_ZONES; i++) {
8061 if (i == ZONE_MOVABLE)
8062 continue;
Anton Blanchardf88dfff2014-12-10 15:42:53 -08008063 pr_info(" %-8s ", zone_names[i]);
David Rientjes72f0ba02010-03-05 13:42:14 -08008064 if (arch_zone_lowest_possible_pfn[i] ==
8065 arch_zone_highest_possible_pfn[i])
Anton Blanchardf88dfff2014-12-10 15:42:53 -08008066 pr_cont("empty\n");
David Rientjes72f0ba02010-03-05 13:42:14 -08008067 else
Juergen Gross8d29e182015-02-11 15:26:01 -08008068 pr_cont("[mem %#018Lx-%#018Lx]\n",
8069 (u64)arch_zone_lowest_possible_pfn[i]
8070 << PAGE_SHIFT,
8071 ((u64)arch_zone_highest_possible_pfn[i]
Bjorn Helgaasa62e2f42012-05-29 15:06:30 -07008072 << PAGE_SHIFT) - 1);
Mel Gorman2a1e2742007-07-17 04:03:12 -07008073 }
8074
8075 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08008076 pr_info("Movable zone start for each node\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07008077 for (i = 0; i < MAX_NUMNODES; i++) {
8078 if (zone_movable_pfn[i])
Juergen Gross8d29e182015-02-11 15:26:01 -08008079 pr_info(" Node %d: %#018Lx\n", i,
8080 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
Mel Gorman2a1e2742007-07-17 04:03:12 -07008081 }
Mel Gormanc7132162006-09-27 01:49:43 -07008082
Dan Williamsf46edbd2019-07-18 15:58:04 -07008083 /*
8084 * Print out the early node map, and initialize the
8085 * subsection-map relative to active online memory ranges to
8086 * enable future "sub-section" extensions of the memory map.
8087 */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08008088 pr_info("Early memory node ranges\n");
Dan Williamsf46edbd2019-07-18 15:58:04 -07008089 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
Juergen Gross8d29e182015-02-11 15:26:01 -08008090 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
8091 (u64)start_pfn << PAGE_SHIFT,
8092 ((u64)end_pfn << PAGE_SHIFT) - 1);
Dan Williamsf46edbd2019-07-18 15:58:04 -07008093 subsection_map_init(start_pfn, end_pfn - start_pfn);
8094 }
Mel Gormanc7132162006-09-27 01:49:43 -07008095
8096 /* Initialise every node */
Mel Gorman708614e2008-07-23 21:26:51 -07008097 mminit_verify_pageflags_layout();
Christoph Lameter8ef82862007-02-20 13:57:52 -08008098 setup_nr_node_ids();
Mel Gormanc7132162006-09-27 01:49:43 -07008099 for_each_online_node(nid) {
8100 pg_data_t *pgdat = NODE_DATA(nid);
Mike Rapoport854e8842020-06-03 15:58:13 -07008101 free_area_init_node(nid);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07008102
8103 /* Any memory on that node */
8104 if (pgdat->node_present_pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08008105 node_set_state(nid, N_MEMORY);
8106 check_for_memory(pgdat, nid);
Mel Gormanc7132162006-09-27 01:49:43 -07008107 }
Mike Rapoport122e0932021-06-28 19:33:26 -07008108
8109 memmap_init();
Mel Gormanc7132162006-09-27 01:49:43 -07008110}
Mel Gorman2a1e2742007-07-17 04:03:12 -07008111
David Rientjesa5c6d652018-04-05 16:23:09 -07008112static int __init cmdline_parse_core(char *p, unsigned long *core,
8113 unsigned long *percent)
Mel Gorman2a1e2742007-07-17 04:03:12 -07008114{
8115 unsigned long long coremem;
David Rientjesa5c6d652018-04-05 16:23:09 -07008116 char *endptr;
8117
Mel Gorman2a1e2742007-07-17 04:03:12 -07008118 if (!p)
8119 return -EINVAL;
8120
David Rientjesa5c6d652018-04-05 16:23:09 -07008121 /* Value may be a percentage of total memory, otherwise bytes */
8122 coremem = simple_strtoull(p, &endptr, 0);
8123 if (*endptr == '%') {
8124 /* Paranoid check for percent values greater than 100 */
8125 WARN_ON(coremem > 100);
Mel Gorman2a1e2742007-07-17 04:03:12 -07008126
David Rientjesa5c6d652018-04-05 16:23:09 -07008127 *percent = coremem;
8128 } else {
8129 coremem = memparse(p, &p);
8130 /* Paranoid check that UL is enough for the coremem value */
8131 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
Mel Gorman2a1e2742007-07-17 04:03:12 -07008132
David Rientjesa5c6d652018-04-05 16:23:09 -07008133 *core = coremem >> PAGE_SHIFT;
8134 *percent = 0UL;
8135 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07008136 return 0;
8137}
Mel Gormaned7ed362007-07-17 04:03:14 -07008138
Mel Gorman7e63efef2007-07-17 04:03:15 -07008139/*
8140 * kernelcore=size sets the amount of memory for use for allocations that
8141 * cannot be reclaimed or migrated.
8142 */
8143static int __init cmdline_parse_kernelcore(char *p)
8144{
Taku Izumi342332e2016-03-15 14:55:22 -07008145 /* parse kernelcore=mirror */
8146 if (parse_option_str(p, "mirror")) {
8147 mirrored_kernelcore = true;
8148 return 0;
8149 }
8150
David Rientjesa5c6d652018-04-05 16:23:09 -07008151 return cmdline_parse_core(p, &required_kernelcore,
8152 &required_kernelcore_percent);
Mel Gorman7e63efef2007-07-17 04:03:15 -07008153}
8154
8155/*
8156 * movablecore=size sets the amount of memory for use for allocations that
8157 * can be reclaimed or migrated.
8158 */
8159static int __init cmdline_parse_movablecore(char *p)
8160{
David Rientjesa5c6d652018-04-05 16:23:09 -07008161 return cmdline_parse_core(p, &required_movablecore,
8162 &required_movablecore_percent);
Mel Gorman7e63efef2007-07-17 04:03:15 -07008163}
8164
Mel Gormaned7ed362007-07-17 04:03:14 -07008165early_param("kernelcore", cmdline_parse_kernelcore);
Mel Gorman7e63efef2007-07-17 04:03:15 -07008166early_param("movablecore", cmdline_parse_movablecore);
Mel Gormaned7ed362007-07-17 04:03:14 -07008167
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07008168void adjust_managed_page_count(struct page *page, long count)
8169{
Arun KS9705bea2018-12-28 00:34:24 -08008170 atomic_long_add(count, &page_zone(page)->managed_pages);
Arun KSca79b0c2018-12-28 00:34:29 -08008171 totalram_pages_add(count);
Jiang Liu3dcc0572013-07-03 15:03:21 -07008172#ifdef CONFIG_HIGHMEM
8173 if (PageHighMem(page))
Arun KSca79b0c2018-12-28 00:34:29 -08008174 totalhigh_pages_add(count);
Jiang Liu3dcc0572013-07-03 15:03:21 -07008175#endif
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07008176}
Jiang Liu3dcc0572013-07-03 15:03:21 -07008177EXPORT_SYMBOL(adjust_managed_page_count);
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07008178
Alexey Dobriyane5cb1132018-12-28 00:36:03 -08008179unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
Jiang Liu69afade2013-04-29 15:06:21 -07008180{
Jiang Liu11199692013-07-03 15:02:48 -07008181 void *pos;
8182 unsigned long pages = 0;
Jiang Liu69afade2013-04-29 15:06:21 -07008183
Jiang Liu11199692013-07-03 15:02:48 -07008184 start = (void *)PAGE_ALIGN((unsigned long)start);
8185 end = (void *)((unsigned long)end & PAGE_MASK);
8186 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
Dave Hansen0d834322018-08-02 15:58:26 -07008187 struct page *page = virt_to_page(pos);
8188 void *direct_map_addr;
8189
8190 /*
8191 * 'direct_map_addr' might be different from 'pos'
8192 * because some architectures' virt_to_page()
8193 * work with aliases. Getting the direct map
8194 * address ensures that we get a _writeable_
8195 * alias for the memset().
8196 */
8197 direct_map_addr = page_address(page);
Vincenzo Frascinoc7461702020-12-22 12:01:49 -08008198 /*
8199 * Perform a kasan-unchecked memset() since this memory
8200 * has not been initialized.
8201 */
8202 direct_map_addr = kasan_reset_tag(direct_map_addr);
Jiang Liudbe67df2013-07-03 15:02:51 -07008203 if ((unsigned int)poison <= 0xFF)
Dave Hansen0d834322018-08-02 15:58:26 -07008204 memset(direct_map_addr, poison, PAGE_SIZE);
8205
8206 free_reserved_page(page);
Jiang Liu69afade2013-04-29 15:06:21 -07008207 }
8208
8209 if (pages && s)
Miaohe Linff7ed9e2021-11-05 13:40:05 -07008210 pr_info("Freeing %s memory: %ldK\n", s, K(pages));
Jiang Liu69afade2013-04-29 15:06:21 -07008211
8212 return pages;
8213}
8214
Kefeng Wang1f9d03c2021-04-29 23:00:55 -07008215void __init mem_init_print_info(void)
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07008216{
8217 unsigned long physpages, codesize, datasize, rosize, bss_size;
8218 unsigned long init_code_size, init_data_size;
8219
8220 physpages = get_num_physpages();
8221 codesize = _etext - _stext;
8222 datasize = _edata - _sdata;
8223 rosize = __end_rodata - __start_rodata;
8224 bss_size = __bss_stop - __bss_start;
8225 init_data_size = __init_end - __init_begin;
8226 init_code_size = _einittext - _sinittext;
8227
8228 /*
8229 * Detect special cases and adjust section sizes accordingly:
8230 * 1) .init.* may be embedded into .data sections
8231 * 2) .init.text.* may be out of [__init_begin, __init_end],
8232 * please refer to arch/tile/kernel/vmlinux.lds.S.
8233 * 3) .rodata.* may be embedded into .text or .data sections.
8234 */
8235#define adj_init_size(start, end, size, pos, adj) \
Pintu Kumarb8af2942013-09-11 14:20:34 -07008236 do { \
Xiongwei Songca831f22022-01-14 14:07:24 -08008237 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
Pintu Kumarb8af2942013-09-11 14:20:34 -07008238 size -= adj; \
8239 } while (0)
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07008240
8241 adj_init_size(__init_begin, __init_end, init_data_size,
8242 _sinittext, init_code_size);
8243 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
8244 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
8245 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
8246 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
8247
8248#undef adj_init_size
8249
Joe Perches756a0252016-03-17 14:19:47 -07008250 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07008251#ifdef CONFIG_HIGHMEM
Joe Perches756a0252016-03-17 14:19:47 -07008252 ", %luK highmem"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07008253#endif
Kefeng Wang1f9d03c2021-04-29 23:00:55 -07008254 ")\n",
Miaohe Linff7ed9e2021-11-05 13:40:05 -07008255 K(nr_free_pages()), K(physpages),
Joe Perches756a0252016-03-17 14:19:47 -07008256 codesize >> 10, datasize >> 10, rosize >> 10,
8257 (init_data_size + init_code_size) >> 10, bss_size >> 10,
Miaohe Linff7ed9e2021-11-05 13:40:05 -07008258 K(physpages - totalram_pages() - totalcma_pages),
8259 K(totalcma_pages)
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07008260#ifdef CONFIG_HIGHMEM
Miaohe Linff7ed9e2021-11-05 13:40:05 -07008261 , K(totalhigh_pages())
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07008262#endif
Kefeng Wang1f9d03c2021-04-29 23:00:55 -07008263 );
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07008264}
8265
Mel Gorman0e0b8642006-09-27 01:49:56 -07008266/**
Randy Dunlap88ca3b92006-10-04 02:15:25 -07008267 * set_dma_reserve - set the specified number of pages reserved in the first zone
8268 * @new_dma_reserve: The number of pages to mark reserved
Mel Gorman0e0b8642006-09-27 01:49:56 -07008269 *
Yaowei Bai013110a2015-09-08 15:04:10 -07008270 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
Mel Gorman0e0b8642006-09-27 01:49:56 -07008271 * In the DMA zone, a significant percentage may be consumed by kernel image
8272 * and other unfreeable allocations which can skew the watermarks badly. This
Randy Dunlap88ca3b92006-10-04 02:15:25 -07008273 * function may optionally be used to account for unfreeable pages in the
8274 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
8275 * smaller per-cpu batchsize.
Mel Gorman0e0b8642006-09-27 01:49:56 -07008276 */
8277void __init set_dma_reserve(unsigned long new_dma_reserve)
8278{
8279 dma_reserve = new_dma_reserve;
8280}
8281
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01008282static int page_alloc_cpu_dead(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008283{
Mel Gorman04f8cfe2021-06-28 19:42:15 -07008284 struct zone *zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008285
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01008286 lru_add_drain_cpu(cpu);
8287 drain_pages(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08008288
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01008289 /*
8290 * Spill the event counters of the dead processor
8291 * into the current processors event counters.
8292 * This artificially elevates the count of the current
8293 * processor.
8294 */
8295 vm_events_fold_cpu(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08008296
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01008297 /*
8298 * Zero the differential counters of the dead processor
8299 * so that the vm statistics are consistent.
8300 *
8301 * This is only okay since the processor is dead and cannot
8302 * race with what we are doing.
8303 */
8304 cpu_vm_stats_fold(cpu);
Mel Gorman04f8cfe2021-06-28 19:42:15 -07008305
8306 for_each_populated_zone(zone)
8307 zone_pcp_update(zone, 0);
8308
8309 return 0;
8310}
8311
8312static int page_alloc_cpu_online(unsigned int cpu)
8313{
8314 struct zone *zone;
8315
8316 for_each_populated_zone(zone)
8317 zone_pcp_update(zone, 1);
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01008318 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008319}
Linus Torvalds1da177e2005-04-16 15:20:36 -07008320
Nicholas Piggine03a5122019-07-11 20:59:12 -07008321#ifdef CONFIG_NUMA
8322int hashdist = HASHDIST_DEFAULT;
8323
8324static int __init set_hashdist(char *str)
8325{
8326 if (!str)
8327 return 0;
8328 hashdist = simple_strtoul(str, &str, 0);
8329 return 1;
8330}
8331__setup("hashdist=", set_hashdist);
8332#endif
8333
Linus Torvalds1da177e2005-04-16 15:20:36 -07008334void __init page_alloc_init(void)
8335{
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01008336 int ret;
8337
Nicholas Piggine03a5122019-07-11 20:59:12 -07008338#ifdef CONFIG_NUMA
8339 if (num_node_state(N_MEMORY) == 1)
8340 hashdist = 0;
8341#endif
8342
Mel Gorman04f8cfe2021-06-28 19:42:15 -07008343 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC,
8344 "mm/page_alloc:pcp",
8345 page_alloc_cpu_online,
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01008346 page_alloc_cpu_dead);
8347 WARN_ON(ret < 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008348}
8349
8350/*
Yaowei Bai34b10062015-09-08 15:04:13 -07008351 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008352 * or min_free_kbytes changes.
8353 */
8354static void calculate_totalreserve_pages(void)
8355{
8356 struct pglist_data *pgdat;
8357 unsigned long reserve_pages = 0;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07008358 enum zone_type i, j;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008359
8360 for_each_online_pgdat(pgdat) {
Mel Gorman281e3722016-07-28 15:46:11 -07008361
8362 pgdat->totalreserve_pages = 0;
8363
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008364 for (i = 0; i < MAX_NR_ZONES; i++) {
8365 struct zone *zone = pgdat->node_zones + i;
Mel Gorman3484b2d2014-08-06 16:07:14 -07008366 long max = 0;
Arun KS9705bea2018-12-28 00:34:24 -08008367 unsigned long managed_pages = zone_managed_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008368
8369 /* Find valid and maximum lowmem_reserve in the zone */
8370 for (j = i; j < MAX_NR_ZONES; j++) {
8371 if (zone->lowmem_reserve[j] > max)
8372 max = zone->lowmem_reserve[j];
8373 }
8374
Mel Gorman41858962009-06-16 15:32:12 -07008375 /* we treat the high watermark as reserved pages. */
8376 max += high_wmark_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008377
Arun KS3d6357d2018-12-28 00:34:20 -08008378 if (max > managed_pages)
8379 max = managed_pages;
Johannes Weinera8d01432016-01-14 15:20:15 -08008380
Mel Gorman281e3722016-07-28 15:46:11 -07008381 pgdat->totalreserve_pages += max;
Johannes Weinera8d01432016-01-14 15:20:15 -08008382
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008383 reserve_pages += max;
8384 }
8385 }
8386 totalreserve_pages = reserve_pages;
8387}
8388
8389/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07008390 * setup_per_zone_lowmem_reserve - called whenever
Yaowei Bai34b10062015-09-08 15:04:13 -07008391 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
Linus Torvalds1da177e2005-04-16 15:20:36 -07008392 * has a correct pages reserved value, so an adequate number of
8393 * pages are left in the zone after a successful __alloc_pages().
8394 */
8395static void setup_per_zone_lowmem_reserve(void)
8396{
8397 struct pglist_data *pgdat;
Lorenzo Stoakes470c61d2020-12-14 19:11:22 -08008398 enum zone_type i, j;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008399
KAMEZAWA Hiroyukiec936fc2006-03-27 01:15:59 -08008400 for_each_online_pgdat(pgdat) {
Lorenzo Stoakes470c61d2020-12-14 19:11:22 -08008401 for (i = 0; i < MAX_NR_ZONES - 1; i++) {
8402 struct zone *zone = &pgdat->node_zones[i];
8403 int ratio = sysctl_lowmem_reserve_ratio[i];
8404 bool clear = !ratio || !zone_managed_pages(zone);
8405 unsigned long managed_pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008406
Lorenzo Stoakes470c61d2020-12-14 19:11:22 -08008407 for (j = i + 1; j < MAX_NR_ZONES; j++) {
Liu Shixinf7ec1042021-06-28 19:42:33 -07008408 struct zone *upper_zone = &pgdat->node_zones[j];
Lorenzo Stoakes470c61d2020-12-14 19:11:22 -08008409
Liu Shixinf7ec1042021-06-28 19:42:33 -07008410 managed_pages += zone_managed_pages(upper_zone);
8411
8412 if (clear)
8413 zone->lowmem_reserve[j] = 0;
8414 else
Lorenzo Stoakes470c61d2020-12-14 19:11:22 -08008415 zone->lowmem_reserve[j] = managed_pages / ratio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008416 }
8417 }
8418 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008419
8420 /* update totalreserve_pages */
8421 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008422}
8423
Mel Gormancfd3da12011-04-25 21:36:42 +00008424static void __setup_per_zone_wmarks(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008425{
8426 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
8427 unsigned long lowmem_pages = 0;
8428 struct zone *zone;
8429 unsigned long flags;
8430
8431 /* Calculate total number of !ZONE_HIGHMEM pages */
8432 for_each_zone(zone) {
8433 if (!is_highmem(zone))
Arun KS9705bea2018-12-28 00:34:24 -08008434 lowmem_pages += zone_managed_pages(zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008435 }
8436
8437 for_each_zone(zone) {
Andrew Mortonac924c62006-05-15 09:43:59 -07008438 u64 tmp;
8439
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07008440 spin_lock_irqsave(&zone->lock, flags);
Arun KS9705bea2018-12-28 00:34:24 -08008441 tmp = (u64)pages_min * zone_managed_pages(zone);
Andrew Mortonac924c62006-05-15 09:43:59 -07008442 do_div(tmp, lowmem_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008443 if (is_highmem(zone)) {
8444 /*
Nick Piggin669ed172005-11-13 16:06:45 -08008445 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
8446 * need highmem pages, so cap pages_min to a small
8447 * value here.
8448 *
Mel Gorman41858962009-06-16 15:32:12 -07008449 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
Wei Yang8bb4e7a2019-03-05 15:46:22 -08008450 * deltas control async page reclaim, and so should
Nick Piggin669ed172005-11-13 16:06:45 -08008451 * not be capped for highmem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008452 */
Andrew Morton90ae8d62013-02-22 16:32:22 -08008453 unsigned long min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008454
Arun KS9705bea2018-12-28 00:34:24 -08008455 min_pages = zone_managed_pages(zone) / 1024;
Andrew Morton90ae8d62013-02-22 16:32:22 -08008456 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
Mel Gormana9214442018-12-28 00:35:44 -08008457 zone->_watermark[WMARK_MIN] = min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008458 } else {
Nick Piggin669ed172005-11-13 16:06:45 -08008459 /*
8460 * If it's a lowmem zone, reserve a number of pages
Linus Torvalds1da177e2005-04-16 15:20:36 -07008461 * proportionate to the zone's size.
8462 */
Mel Gormana9214442018-12-28 00:35:44 -08008463 zone->_watermark[WMARK_MIN] = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008464 }
8465
Johannes Weiner795ae7a2016-03-17 14:19:14 -07008466 /*
8467 * Set the kswapd watermarks distance according to the
8468 * scale factor in proportion to available memory, but
8469 * ensure a minimum size on small systems.
8470 */
8471 tmp = max_t(u64, tmp >> 2,
Arun KS9705bea2018-12-28 00:34:24 -08008472 mult_frac(zone_managed_pages(zone),
Johannes Weiner795ae7a2016-03-17 14:19:14 -07008473 watermark_scale_factor, 10000));
8474
Charan Teja Reddyaa092592020-06-03 15:59:14 -07008475 zone->watermark_boost = 0;
Mel Gormana9214442018-12-28 00:35:44 -08008476 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
8477 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
Marek Szyprowski49f223a2012-01-25 12:49:24 +01008478
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07008479 spin_unlock_irqrestore(&zone->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008480 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07008481
8482 /* update totalreserve_pages */
8483 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008484}
8485
Mel Gormancfd3da12011-04-25 21:36:42 +00008486/**
8487 * setup_per_zone_wmarks - called when min_free_kbytes changes
8488 * or when memory is hot-{added|removed}
8489 *
8490 * Ensures that the watermark[min,low,high] values for each zone are set
8491 * correctly with respect to min_free_kbytes.
8492 */
8493void setup_per_zone_wmarks(void)
8494{
Mel Gormanb92ca182021-06-28 19:42:12 -07008495 struct zone *zone;
Michal Hockob93e0f32017-09-06 16:20:37 -07008496 static DEFINE_SPINLOCK(lock);
8497
8498 spin_lock(&lock);
Mel Gormancfd3da12011-04-25 21:36:42 +00008499 __setup_per_zone_wmarks();
Michal Hockob93e0f32017-09-06 16:20:37 -07008500 spin_unlock(&lock);
Mel Gormanb92ca182021-06-28 19:42:12 -07008501
8502 /*
8503 * The watermark size have changed so update the pcpu batch
8504 * and high limits or the limits may be inappropriate.
8505 */
8506 for_each_zone(zone)
Mel Gorman04f8cfe2021-06-28 19:42:15 -07008507 zone_pcp_update(zone, 0);
Mel Gormancfd3da12011-04-25 21:36:42 +00008508}
8509
Randy Dunlap55a44622009-09-21 17:01:20 -07008510/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07008511 * Initialise min_free_kbytes.
8512 *
8513 * For small machines we want it small (128k min). For large machines
Joel Savitz8beeae82020-07-03 15:15:30 -07008514 * we want it large (256MB max). But it is not linear, because network
Linus Torvalds1da177e2005-04-16 15:20:36 -07008515 * bandwidth does not increase linearly with machine size. We use
8516 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07008517 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
Linus Torvalds1da177e2005-04-16 15:20:36 -07008518 * min_free_kbytes = sqrt(lowmem_kbytes * 16)
8519 *
8520 * which yields
8521 *
8522 * 16MB: 512k
8523 * 32MB: 724k
8524 * 64MB: 1024k
8525 * 128MB: 1448k
8526 * 256MB: 2048k
8527 * 512MB: 2896k
8528 * 1024MB: 4096k
8529 * 2048MB: 5792k
8530 * 4096MB: 8192k
8531 * 8192MB: 11584k
8532 * 16384MB: 16384k
8533 */
Liangcai Fanbd3400e2021-11-05 13:41:36 -07008534void calculate_min_free_kbytes(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008535{
8536 unsigned long lowmem_kbytes;
Michal Hocko5f127332013-07-08 16:00:40 -07008537 int new_min_free_kbytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008538
8539 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
Michal Hocko5f127332013-07-08 16:00:40 -07008540 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008541
Wang ShaoBo59d336b2021-11-05 13:40:55 -07008542 if (new_min_free_kbytes > user_min_free_kbytes)
8543 min_free_kbytes = clamp(new_min_free_kbytes, 128, 262144);
8544 else
Michal Hocko5f127332013-07-08 16:00:40 -07008545 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
8546 new_min_free_kbytes, user_min_free_kbytes);
Wang ShaoBo59d336b2021-11-05 13:40:55 -07008547
Liangcai Fanbd3400e2021-11-05 13:41:36 -07008548}
8549
8550int __meminit init_per_zone_wmark_min(void)
8551{
8552 calculate_min_free_kbytes();
Minchan Kimbc75d332009-06-16 15:32:48 -07008553 setup_per_zone_wmarks();
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -07008554 refresh_zone_stat_thresholds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008555 setup_per_zone_lowmem_reserve();
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008556
8557#ifdef CONFIG_NUMA
8558 setup_min_unmapped_ratio();
8559 setup_min_slab_ratio();
8560#endif
8561
Vijay Balakrishna4aab2be2020-10-10 23:16:40 -07008562 khugepaged_min_free_kbytes_update();
8563
Linus Torvalds1da177e2005-04-16 15:20:36 -07008564 return 0;
8565}
Doug Bergere08d3fd2020-08-20 17:42:24 -07008566postcore_initcall(init_per_zone_wmark_min)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008567
8568/*
Pintu Kumarb8af2942013-09-11 14:20:34 -07008569 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
Linus Torvalds1da177e2005-04-16 15:20:36 -07008570 * that we can call two helper functions whenever min_free_kbytes
8571 * changes.
8572 */
Joe Perchescccad5b2014-06-06 14:38:09 -07008573int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008574 void *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008575{
Han Pingtianda8c7572014-01-23 15:53:17 -08008576 int rc;
8577
8578 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
8579 if (rc)
8580 return rc;
8581
Michal Hocko5f127332013-07-08 16:00:40 -07008582 if (write) {
8583 user_min_free_kbytes = min_free_kbytes;
Minchan Kimbc75d332009-06-16 15:32:48 -07008584 setup_per_zone_wmarks();
Michal Hocko5f127332013-07-08 16:00:40 -07008585 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008586 return 0;
8587}
8588
Johannes Weiner795ae7a2016-03-17 14:19:14 -07008589int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008590 void *buffer, size_t *length, loff_t *ppos)
Johannes Weiner795ae7a2016-03-17 14:19:14 -07008591{
8592 int rc;
8593
8594 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
8595 if (rc)
8596 return rc;
8597
8598 if (write)
8599 setup_per_zone_wmarks();
8600
8601 return 0;
8602}
8603
Christoph Lameter96146342006-07-03 00:24:13 -07008604#ifdef CONFIG_NUMA
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008605static void setup_min_unmapped_ratio(void)
Christoph Lameter96146342006-07-03 00:24:13 -07008606{
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008607 pg_data_t *pgdat;
Christoph Lameter96146342006-07-03 00:24:13 -07008608 struct zone *zone;
Christoph Lameter96146342006-07-03 00:24:13 -07008609
Mel Gormana5f5f912016-07-28 15:46:32 -07008610 for_each_online_pgdat(pgdat)
Joonsoo Kim81cbcbc2016-08-10 16:27:46 -07008611 pgdat->min_unmapped_pages = 0;
Mel Gormana5f5f912016-07-28 15:46:32 -07008612
Christoph Lameter96146342006-07-03 00:24:13 -07008613 for_each_zone(zone)
Arun KS9705bea2018-12-28 00:34:24 -08008614 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
8615 sysctl_min_unmapped_ratio) / 100;
Christoph Lameter96146342006-07-03 00:24:13 -07008616}
Christoph Lameter0ff38492006-09-25 23:31:52 -07008617
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008618
8619int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008620 void *buffer, size_t *length, loff_t *ppos)
Christoph Lameter0ff38492006-09-25 23:31:52 -07008621{
Christoph Lameter0ff38492006-09-25 23:31:52 -07008622 int rc;
8623
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008624 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
Christoph Lameter0ff38492006-09-25 23:31:52 -07008625 if (rc)
8626 return rc;
8627
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008628 setup_min_unmapped_ratio();
8629
8630 return 0;
8631}
8632
8633static void setup_min_slab_ratio(void)
8634{
8635 pg_data_t *pgdat;
8636 struct zone *zone;
8637
Mel Gormana5f5f912016-07-28 15:46:32 -07008638 for_each_online_pgdat(pgdat)
8639 pgdat->min_slab_pages = 0;
8640
Christoph Lameter0ff38492006-09-25 23:31:52 -07008641 for_each_zone(zone)
Arun KS9705bea2018-12-28 00:34:24 -08008642 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
8643 sysctl_min_slab_ratio) / 100;
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008644}
8645
8646int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008647 void *buffer, size_t *length, loff_t *ppos)
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008648{
8649 int rc;
8650
8651 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
8652 if (rc)
8653 return rc;
8654
8655 setup_min_slab_ratio();
8656
Christoph Lameter0ff38492006-09-25 23:31:52 -07008657 return 0;
8658}
Christoph Lameter96146342006-07-03 00:24:13 -07008659#endif
8660
Linus Torvalds1da177e2005-04-16 15:20:36 -07008661/*
8662 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
8663 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
8664 * whenever sysctl_lowmem_reserve_ratio changes.
8665 *
8666 * The reserve ratio obviously has absolutely no relation with the
Mel Gorman41858962009-06-16 15:32:12 -07008667 * minimum watermarks. The lowmem reserve ratio can only make sense
Linus Torvalds1da177e2005-04-16 15:20:36 -07008668 * if in function of the boot time zone sizes.
8669 */
Joe Perchescccad5b2014-06-06 14:38:09 -07008670int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008671 void *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008672{
Baoquan He86aaf252020-06-03 15:58:48 -07008673 int i;
8674
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008675 proc_dointvec_minmax(table, write, buffer, length, ppos);
Baoquan He86aaf252020-06-03 15:58:48 -07008676
8677 for (i = 0; i < MAX_NR_ZONES; i++) {
8678 if (sysctl_lowmem_reserve_ratio[i] < 1)
8679 sysctl_lowmem_reserve_ratio[i] = 0;
8680 }
8681
Linus Torvalds1da177e2005-04-16 15:20:36 -07008682 setup_per_zone_lowmem_reserve();
8683 return 0;
8684}
8685
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008686/*
Mel Gorman74f44822021-06-28 19:42:24 -07008687 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each
8688 * cpu. It is the fraction of total pages in each zone that a hot per cpu
Pintu Kumarb8af2942013-09-11 14:20:34 -07008689 * pagelist can have before it gets flushed back to buddy allocator.
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008690 */
Mel Gorman74f44822021-06-28 19:42:24 -07008691int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *table,
8692 int write, void *buffer, size_t *length, loff_t *ppos)
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008693{
8694 struct zone *zone;
Mel Gorman74f44822021-06-28 19:42:24 -07008695 int old_percpu_pagelist_high_fraction;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008696 int ret;
8697
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008698 mutex_lock(&pcp_batch_high_lock);
Mel Gorman74f44822021-06-28 19:42:24 -07008699 old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction;
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008700
8701 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
8702 if (!write || ret < 0)
8703 goto out;
8704
8705 /* Sanity checking to avoid pcp imbalance */
Mel Gorman74f44822021-06-28 19:42:24 -07008706 if (percpu_pagelist_high_fraction &&
8707 percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) {
8708 percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction;
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008709 ret = -EINVAL;
8710 goto out;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008711 }
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008712
8713 /* No change? */
Mel Gorman74f44822021-06-28 19:42:24 -07008714 if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction)
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008715 goto out;
8716
Mel Gormancb1ef532019-11-30 17:55:11 -08008717 for_each_populated_zone(zone)
Mel Gorman74f44822021-06-28 19:42:24 -07008718 zone_set_pageset_high_and_batch(zone, 0);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008719out:
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008720 mutex_unlock(&pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008721 return ret;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008722}
8723
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07008724#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
8725/*
8726 * Returns the number of pages that arch has reserved but
8727 * is not known to alloc_large_system_hash().
8728 */
8729static unsigned long __init arch_reserved_kernel_pages(void)
8730{
8731 return 0;
8732}
8733#endif
8734
Linus Torvalds1da177e2005-04-16 15:20:36 -07008735/*
Pavel Tatashin90172172017-07-06 15:39:14 -07008736 * Adaptive scale is meant to reduce sizes of hash tables on large memory
8737 * machines. As memory size is increased the scale is also increased but at
8738 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
8739 * quadruples the scale is increased by one, which means the size of hash table
8740 * only doubles, instead of quadrupling as well.
8741 * Because 32-bit systems cannot have large physical memory, where this scaling
8742 * makes sense, it is disabled on such platforms.
8743 */
8744#if __BITS_PER_LONG > 32
8745#define ADAPT_SCALE_BASE (64ul << 30)
8746#define ADAPT_SCALE_SHIFT 2
8747#define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
8748#endif
8749
8750/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07008751 * allocate a large system hash table from bootmem
8752 * - it is assumed that the hash table must contain an exact power-of-2
8753 * quantity of entries
8754 * - limit is the number of hash buckets, not the total allocation size
8755 */
8756void *__init alloc_large_system_hash(const char *tablename,
8757 unsigned long bucketsize,
8758 unsigned long numentries,
8759 int scale,
8760 int flags,
8761 unsigned int *_hash_shift,
8762 unsigned int *_hash_mask,
Tim Bird31fe62b2012-05-23 13:33:35 +00008763 unsigned long low_limit,
8764 unsigned long high_limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008765{
Tim Bird31fe62b2012-05-23 13:33:35 +00008766 unsigned long long max = high_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008767 unsigned long log2qty, size;
8768 void *table = NULL;
Pavel Tatashin3749a8f2017-07-06 15:39:08 -07008769 gfp_t gfp_flags;
Nicholas Pigginec114082019-07-11 20:59:09 -07008770 bool virt;
Nicholas Piggin121e6f32021-04-29 22:58:49 -07008771 bool huge;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008772
8773 /* allow the kernel cmdline to have a say */
8774 if (!numentries) {
8775 /* round applicable memory size up to nearest megabyte */
Andrew Morton04903662006-12-06 20:37:33 -08008776 numentries = nr_kernel_pages;
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07008777 numentries -= arch_reserved_kernel_pages();
Jerry Zhoua7e83312013-09-11 14:20:26 -07008778
8779 /* It isn't necessary when PAGE_SIZE >= 1MB */
8780 if (PAGE_SHIFT < 20)
8781 numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008782
Pavel Tatashin90172172017-07-06 15:39:14 -07008783#if __BITS_PER_LONG > 32
8784 if (!high_limit) {
8785 unsigned long adapt;
8786
8787 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
8788 adapt <<= ADAPT_SCALE_SHIFT)
8789 scale++;
8790 }
8791#endif
8792
Linus Torvalds1da177e2005-04-16 15:20:36 -07008793 /* limit to 1 bucket per 2^scale bytes of low memory */
8794 if (scale > PAGE_SHIFT)
8795 numentries >>= (scale - PAGE_SHIFT);
8796 else
8797 numentries <<= (PAGE_SHIFT - scale);
Paul Mundt9ab37b82007-01-05 16:36:30 -08008798
8799 /* Make sure we've got at least a 0-order allocation.. */
Jan Beulich2c85f512009-09-21 17:03:07 -07008800 if (unlikely(flags & HASH_SMALL)) {
8801 /* Makes no sense without HASH_EARLY */
8802 WARN_ON(!(flags & HASH_EARLY));
8803 if (!(numentries >> *_hash_shift)) {
8804 numentries = 1UL << *_hash_shift;
8805 BUG_ON(!numentries);
8806 }
8807 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
Paul Mundt9ab37b82007-01-05 16:36:30 -08008808 numentries = PAGE_SIZE / bucketsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008809 }
John Hawkes6e692ed2006-03-25 03:08:02 -08008810 numentries = roundup_pow_of_two(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008811
8812 /* limit allocation size to 1/16 total memory by default */
8813 if (max == 0) {
8814 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
8815 do_div(max, bucketsize);
8816 }
Dimitri Sivanich074b8512012-02-08 12:39:07 -08008817 max = min(max, 0x80000000ULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008818
Tim Bird31fe62b2012-05-23 13:33:35 +00008819 if (numentries < low_limit)
8820 numentries = low_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008821 if (numentries > max)
8822 numentries = max;
8823
David Howellsf0d1b0b2006-12-08 02:37:49 -08008824 log2qty = ilog2(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008825
Pavel Tatashin3749a8f2017-07-06 15:39:08 -07008826 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008827 do {
Nicholas Pigginec114082019-07-11 20:59:09 -07008828 virt = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008829 size = bucketsize << log2qty;
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008830 if (flags & HASH_EARLY) {
8831 if (flags & HASH_ZERO)
Mike Rapoport26fb3da2019-03-11 23:30:42 -07008832 table = memblock_alloc(size, SMP_CACHE_BYTES);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008833 else
Mike Rapoport7e1c4e22018-10-30 15:09:57 -07008834 table = memblock_alloc_raw(size,
8835 SMP_CACHE_BYTES);
Nicholas Pigginec114082019-07-11 20:59:09 -07008836 } else if (get_order(size) >= MAX_ORDER || hashdist) {
Christoph Hellwig88dca4c2020-06-01 21:51:40 -07008837 table = __vmalloc(size, gfp_flags);
Nicholas Pigginec114082019-07-11 20:59:09 -07008838 virt = true;
Eric Dumazet084f7e22021-11-05 13:39:59 -07008839 if (table)
8840 huge = is_vm_area_hugepages(table);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008841 } else {
Eric Dumazet1037b832007-07-15 23:38:05 -07008842 /*
8843 * If bucketsize is not a power-of-two, we may free
Mel Gormana1dd2682009-06-16 15:32:19 -07008844 * some pages at the end of hash table which
8845 * alloc_pages_exact() automatically does
Eric Dumazet1037b832007-07-15 23:38:05 -07008846 */
Nicholas Pigginec114082019-07-11 20:59:09 -07008847 table = alloc_pages_exact(size, gfp_flags);
8848 kmemleak_alloc(table, size, 1, gfp_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008849 }
8850 } while (!table && size > PAGE_SIZE && --log2qty);
8851
8852 if (!table)
8853 panic("Failed to allocate %s hash table\n", tablename);
8854
Nicholas Pigginec114082019-07-11 20:59:09 -07008855 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
8856 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
Nicholas Piggin121e6f32021-04-29 22:58:49 -07008857 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");
Linus Torvalds1da177e2005-04-16 15:20:36 -07008858
8859 if (_hash_shift)
8860 *_hash_shift = log2qty;
8861 if (_hash_mask)
8862 *_hash_mask = (1 << log2qty) - 1;
8863
8864 return table;
8865}
KAMEZAWA Hiroyukia117e662006-03-27 01:15:25 -08008866
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -07008867/*
Minchan Kim80934512012-07-31 16:43:01 -07008868 * This function checks whether pageblock includes unmovable pages or not.
Minchan Kim80934512012-07-31 16:43:01 -07008869 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07008870 * PageLRU check without isolation or lru_lock could race so that
Yisheng Xie0efadf42017-02-24 14:57:39 -08008871 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
8872 * check without lock_page also may miss some movable non-lru pages at
8873 * race condition. So you can't expect this function should be exact.
Qian Cai4a55c042020-01-30 22:14:57 -08008874 *
8875 * Returns a page without holding a reference. If the caller wants to
Randy Dunlap047b9962020-08-11 18:33:14 -07008876 * dereference that page (e.g., dumping), it has to make sure that it
Qian Cai4a55c042020-01-30 22:14:57 -08008877 * cannot get removed (e.g., via memory unplug) concurrently.
8878 *
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -07008879 */
Qian Cai4a55c042020-01-30 22:14:57 -08008880struct page *has_unmovable_pages(struct zone *zone, struct page *page,
8881 int migratetype, int flags)
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008882{
Qian Cai1a9f21912019-04-18 17:50:30 -07008883 unsigned long iter = 0;
8884 unsigned long pfn = page_to_pfn(page);
Li Xinhai6a654e32020-10-13 16:55:39 -07008885 unsigned long offset = pfn % pageblock_nr_pages;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01008886
Qian Cai1a9f21912019-04-18 17:50:30 -07008887 if (is_migrate_cma_page(page)) {
8888 /*
8889 * CMA allocations (alloc_contig_range) really need to mark
8890 * isolate CMA pageblocks even when they are not movable in fact
8891 * so consider them movable here.
8892 */
8893 if (is_migrate_cma(migratetype))
Qian Cai4a55c042020-01-30 22:14:57 -08008894 return NULL;
Michal Hocko4da2ce22017-11-15 17:33:26 -08008895
Qian Cai3d680bd2020-01-30 22:15:01 -08008896 return page;
Qian Cai1a9f21912019-04-18 17:50:30 -07008897 }
8898
Li Xinhai6a654e32020-10-13 16:55:39 -07008899 for (; iter < pageblock_nr_pages - offset; iter++) {
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008900 page = pfn_to_page(pfn + iter);
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008901
David Hildenbrandc9c510d2020-10-13 16:55:17 -07008902 /*
8903 * Both, bootmem allocations and memory holes are marked
8904 * PG_reserved and are unmovable. We can even have unmovable
8905 * allocations inside ZONE_MOVABLE, for example when
8906 * specifying "movablecore".
8907 */
Michal Hockod7ab3672017-11-15 17:33:30 -08008908 if (PageReserved(page))
Qian Cai3d680bd2020-01-30 22:15:01 -08008909 return page;
Michal Hockod7ab3672017-11-15 17:33:30 -08008910
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008911 /*
Michal Hocko9d789992018-11-16 15:08:15 -08008912 * If the zone is movable and we have ruled out all reserved
8913 * pages then it should be reasonably safe to assume the rest
8914 * is movable.
8915 */
8916 if (zone_idx(zone) == ZONE_MOVABLE)
8917 continue;
8918
8919 /*
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008920 * Hugepages are not in LRU lists, but they're movable.
Rik van Riel1da2f322020-04-01 21:10:31 -07008921 * THPs are on the LRU, but need to be counted as #small pages.
Wei Yang8bb4e7a2019-03-05 15:46:22 -08008922 * We need not scan over tail pages because we don't
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008923 * handle each tail page individually in migration.
8924 */
Rik van Riel1da2f322020-04-01 21:10:31 -07008925 if (PageHuge(page) || PageTransCompound(page)) {
Oscar Salvador17e2e7d2018-12-21 14:31:00 -08008926 struct page *head = compound_head(page);
8927 unsigned int skip_pages;
Aneesh Kumar K.V464c7ff2018-09-04 15:45:59 -07008928
Rik van Riel1da2f322020-04-01 21:10:31 -07008929 if (PageHuge(page)) {
8930 if (!hugepage_migration_supported(page_hstate(head)))
8931 return page;
8932 } else if (!PageLRU(head) && !__PageMovable(head)) {
Qian Cai3d680bd2020-01-30 22:15:01 -08008933 return page;
Rik van Riel1da2f322020-04-01 21:10:31 -07008934 }
Aneesh Kumar K.V464c7ff2018-09-04 15:45:59 -07008935
Matthew Wilcox (Oracle)d8c65462019-09-23 15:34:30 -07008936 skip_pages = compound_nr(head) - (page - head);
Oscar Salvador17e2e7d2018-12-21 14:31:00 -08008937 iter += skip_pages - 1;
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008938 continue;
8939 }
8940
Minchan Kim97d255c2012-07-31 16:42:59 -07008941 /*
8942 * We can't use page_count without pin a page
8943 * because another CPU can free compound page.
8944 * This check already skips compound tails of THP
Joonsoo Kim0139aa72016-05-19 17:10:49 -07008945 * because their page->_refcount is zero at all time.
Minchan Kim97d255c2012-07-31 16:42:59 -07008946 */
Joonsoo Kimfe896d12016-03-17 14:19:26 -07008947 if (!page_ref_count(page)) {
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008948 if (PageBuddy(page))
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008949 iter += (1 << buddy_order(page)) - 1;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008950 continue;
8951 }
Minchan Kim97d255c2012-07-31 16:42:59 -07008952
Wen Congyangb023f462012-12-11 16:00:45 -08008953 /*
8954 * The HWPoisoned page may be not in buddy system, and
8955 * page_count() is not 0.
8956 */
David Hildenbrand756d25b2019-11-30 17:54:07 -08008957 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
Wen Congyangb023f462012-12-11 16:00:45 -08008958 continue;
8959
David Hildenbrandaa218792020-05-07 16:01:30 +02008960 /*
8961 * We treat all PageOffline() pages as movable when offlining
8962 * to give drivers a chance to decrement their reference count
8963 * in MEM_GOING_OFFLINE in order to indicate that these pages
8964 * can be offlined as there are no direct references anymore.
8965 * For actually unmovable PageOffline() where the driver does
8966 * not support this, we will fail later when trying to actually
8967 * move these pages that still have a reference count > 0.
8968 * (false negatives in this function only)
8969 */
8970 if ((flags & MEMORY_OFFLINE) && PageOffline(page))
8971 continue;
8972
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008973 if (__PageMovable(page) || PageLRU(page))
Yisheng Xie0efadf42017-02-24 14:57:39 -08008974 continue;
8975
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008976 /*
Johannes Weiner6b4f7792014-12-12 16:56:13 -08008977 * If there are RECLAIMABLE pages, we need to check
8978 * it. But now, memory offline itself doesn't call
8979 * shrink_node_slabs() and it still to be fixed.
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008980 */
Qian Cai3d680bd2020-01-30 22:15:01 -08008981 return page;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008982 }
Qian Cai4a55c042020-01-30 22:14:57 -08008983 return NULL;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008984}
8985
Alexandre Ghiti8df995f2019-05-13 17:19:00 -07008986#ifdef CONFIG_CONTIG_ALLOC
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008987static unsigned long pfn_max_align_down(unsigned long pfn)
8988{
8989 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
8990 pageblock_nr_pages) - 1);
8991}
8992
8993static unsigned long pfn_max_align_up(unsigned long pfn)
8994{
8995 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
8996 pageblock_nr_pages));
8997}
8998
Minchan Kima1394bd2021-04-29 23:01:30 -07008999#if defined(CONFIG_DYNAMIC_DEBUG) || \
9000 (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
9001/* Usage: See admin-guide/dynamic-debug-howto.rst */
9002static void alloc_contig_dump_pages(struct list_head *page_list)
9003{
9004 DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure");
9005
9006 if (DYNAMIC_DEBUG_BRANCH(descriptor)) {
9007 struct page *page;
9008
9009 dump_stack();
9010 list_for_each_entry(page, page_list, lru)
9011 dump_page(page, "migration failure");
9012 }
9013}
9014#else
9015static inline void alloc_contig_dump_pages(struct list_head *page_list)
9016{
9017}
9018#endif
9019
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009020/* [start, end) must belong to a single zone. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009021static int __alloc_contig_migrate_range(struct compact_control *cc,
9022 unsigned long start, unsigned long end)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009023{
9024 /* This function is based on compact_zone() from compaction.c. */
Maninder Singh730ec8c2020-06-03 16:01:18 -07009025 unsigned int nr_reclaimed;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009026 unsigned long pfn = start;
9027 unsigned int tries = 0;
9028 int ret = 0;
Joonsoo Kim8b94e0b2020-08-11 18:37:31 -07009029 struct migration_target_control mtc = {
9030 .nid = zone_to_nid(cc->zone),
9031 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
9032 };
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009033
Minchan Kim361a2a22021-05-04 18:36:57 -07009034 lru_cache_disable();
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009035
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009036 while (pfn < end || !list_empty(&cc->migratepages)) {
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009037 if (fatal_signal_pending(current)) {
9038 ret = -EINTR;
9039 break;
9040 }
9041
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009042 if (list_empty(&cc->migratepages)) {
9043 cc->nr_migratepages = 0;
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07009044 ret = isolate_migratepages_range(cc, pfn, end);
9045 if (ret && ret != -EAGAIN)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009046 break;
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07009047 pfn = cc->migrate_pfn;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009048 tries = 0;
9049 } else if (++tries == 5) {
Oscar Salvadorc8e28b42021-05-04 18:35:14 -07009050 ret = -EBUSY;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009051 break;
9052 }
9053
Minchan Kimbeb51ea2012-10-08 16:33:51 -07009054 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
9055 &cc->migratepages);
9056 cc->nr_migratepages -= nr_reclaimed;
Minchan Kim02c6de82012-10-08 16:31:55 -07009057
Joonsoo Kim8b94e0b2020-08-11 18:37:31 -07009058 ret = migrate_pages(&cc->migratepages, alloc_migration_target,
Yang Shi5ac95882021-09-02 14:59:13 -07009059 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL);
Oscar Salvadorc8e28b42021-05-04 18:35:14 -07009060
9061 /*
9062 * On -ENOMEM, migrate_pages() bails out right away. It is pointless
9063 * to retry again over this error, so do the same here.
9064 */
9065 if (ret == -ENOMEM)
9066 break;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009067 }
Minchan Kimd479960e2021-05-04 18:36:54 -07009068
Minchan Kim361a2a22021-05-04 18:36:57 -07009069 lru_cache_enable();
Srinivas Pandruvada2a6f5122013-02-22 16:32:09 -08009070 if (ret < 0) {
Minchan Kim151e084a2021-06-28 19:42:06 -07009071 if (ret == -EBUSY)
9072 alloc_contig_dump_pages(&cc->migratepages);
Srinivas Pandruvada2a6f5122013-02-22 16:32:09 -08009073 putback_movable_pages(&cc->migratepages);
9074 return ret;
9075 }
9076 return 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009077}
9078
9079/**
9080 * alloc_contig_range() -- tries to allocate given range of pages
9081 * @start: start PFN to allocate
9082 * @end: one-past-the-last PFN to allocate
Ingo Molnarf0953a12021-05-06 18:06:47 -07009083 * @migratetype: migratetype of the underlying pageblocks (either
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02009084 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
9085 * in range must have the same migratetype and it must
9086 * be either of the two.
Lucas Stachca96b622017-02-24 14:58:37 -08009087 * @gfp_mask: GFP mask to use during compaction
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009088 *
9089 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
Mike Kravetz2c7452a2018-04-05 16:25:26 -07009090 * aligned. The PFN range must belong to a single zone.
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009091 *
Mike Kravetz2c7452a2018-04-05 16:25:26 -07009092 * The first thing this routine does is attempt to MIGRATE_ISOLATE all
9093 * pageblocks in the range. Once isolated, the pageblocks should not
9094 * be modified by others.
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009095 *
Mike Rapoporta862f682019-03-05 15:48:42 -08009096 * Return: zero on success or negative error code. On success all
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009097 * pages which PFN is in [start, end) are allocated for the caller and
9098 * need to be freed with free_contig_range().
9099 */
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02009100int alloc_contig_range(unsigned long start, unsigned long end,
Lucas Stachca96b622017-02-24 14:58:37 -08009101 unsigned migratetype, gfp_t gfp_mask)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009102{
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009103 unsigned long outer_start, outer_end;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08009104 unsigned int order;
9105 int ret = 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009106
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009107 struct compact_control cc = {
9108 .nr_migratepages = 0,
9109 .order = -1,
9110 .zone = page_zone(pfn_to_page(start)),
David Rientjese0b9dae2014-06-04 16:08:28 -07009111 .mode = MIGRATE_SYNC,
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009112 .ignore_skip_hint = true,
Vlastimil Babka2583d672017-11-17 15:26:38 -08009113 .no_set_skip_hint = true,
Michal Hocko7dea19f2017-05-03 14:53:15 -07009114 .gfp_mask = current_gfp_context(gfp_mask),
Rik van Rielb06eda02020-04-01 21:10:28 -07009115 .alloc_contig = true,
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009116 };
9117 INIT_LIST_HEAD(&cc.migratepages);
9118
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009119 /*
9120 * What we do here is we mark all pageblocks in range as
9121 * MIGRATE_ISOLATE. Because pageblock and max order pages may
9122 * have different sizes, and due to the way page allocator
9123 * work, we align the range to biggest of the two pages so
9124 * that page allocator won't try to merge buddies from
9125 * different pageblocks and change MIGRATE_ISOLATE to some
9126 * other migration type.
9127 *
9128 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
9129 * migrate the pages from an unaligned range (ie. pages that
9130 * we are interested in). This will put all the pages in
9131 * range back to page allocator as MIGRATE_ISOLATE.
9132 *
9133 * When this is done, we take the pages in range from page
9134 * allocator removing them from the buddy system. This way
9135 * page allocator will never consider using them.
9136 *
9137 * This lets us mark the pageblocks back as
9138 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
9139 * aligned range but not in the unaligned, original range are
9140 * put back to page allocator so that buddy can use them.
9141 */
9142
9143 ret = start_isolate_page_range(pfn_max_align_down(start),
Michal Hockod381c542018-12-28 00:33:56 -08009144 pfn_max_align_up(end), migratetype, 0);
David Hildenbrand3fa0c7c2020-10-15 20:08:07 -07009145 if (ret)
Bob Liu86a595f2012-10-25 13:37:56 -07009146 return ret;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009147
Vlastimil Babka76129212020-12-14 19:10:56 -08009148 drain_all_pages(cc.zone);
9149
Joonsoo Kim8ef58492016-01-14 15:18:45 -08009150 /*
9151 * In case of -EBUSY, we'd like to know which page causes problem.
Mike Kravetz63cd4482017-11-29 16:10:01 -08009152 * So, just fall through. test_pages_isolated() has a tracepoint
9153 * which will report the busy page.
9154 *
9155 * It is possible that busy pages could become available before
9156 * the call to test_pages_isolated, and the range will actually be
9157 * allocated. So, if we fall through be sure to clear ret so that
9158 * -EBUSY is not accidentally used or returned to caller.
Joonsoo Kim8ef58492016-01-14 15:18:45 -08009159 */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009160 ret = __alloc_contig_migrate_range(&cc, start, end);
Joonsoo Kim8ef58492016-01-14 15:18:45 -08009161 if (ret && ret != -EBUSY)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009162 goto done;
Zhiyuan Dai68d68ff2021-05-04 18:40:12 -07009163 ret = 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009164
9165 /*
9166 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
9167 * aligned blocks that are marked as MIGRATE_ISOLATE. What's
9168 * more, all pages in [start, end) are free in page allocator.
9169 * What we are going to do is to allocate all pages from
9170 * [start, end) (that is remove them from page allocator).
9171 *
9172 * The only problem is that pages at the beginning and at the
9173 * end of interesting range may be not aligned with pages that
9174 * page allocator holds, ie. they can be part of higher order
9175 * pages. Because of this, we reserve the bigger range and
9176 * once this is done free the pages we are not interested in.
9177 *
9178 * We don't have to hold zone->lock here because the pages are
9179 * isolated thus they won't get removed from buddy.
9180 */
9181
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009182 order = 0;
9183 outer_start = start;
9184 while (!PageBuddy(pfn_to_page(outer_start))) {
9185 if (++order >= MAX_ORDER) {
Joonsoo Kim8ef58492016-01-14 15:18:45 -08009186 outer_start = start;
9187 break;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009188 }
9189 outer_start &= ~0UL << order;
9190 }
9191
Joonsoo Kim8ef58492016-01-14 15:18:45 -08009192 if (outer_start != start) {
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07009193 order = buddy_order(pfn_to_page(outer_start));
Joonsoo Kim8ef58492016-01-14 15:18:45 -08009194
9195 /*
9196 * outer_start page could be small order buddy page and
9197 * it doesn't include start page. Adjust outer_start
9198 * in this case to report failed page properly
9199 * on tracepoint in test_pages_isolated()
9200 */
9201 if (outer_start + (1UL << order) <= start)
9202 outer_start = start;
9203 }
9204
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009205 /* Make sure the range is really isolated. */
David Hildenbrand756d25b2019-11-30 17:54:07 -08009206 if (test_pages_isolated(outer_start, end, 0)) {
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009207 ret = -EBUSY;
9208 goto done;
9209 }
9210
Marek Szyprowski49f223a2012-01-25 12:49:24 +01009211 /* Grab isolated pages from freelists. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07009212 outer_end = isolate_freepages_range(&cc, outer_start, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009213 if (!outer_end) {
9214 ret = -EBUSY;
9215 goto done;
9216 }
9217
9218 /* Free head and tail (if any) */
9219 if (start != outer_start)
9220 free_contig_range(outer_start, start - outer_start);
9221 if (end != outer_end)
9222 free_contig_range(end, outer_end - end);
9223
9224done:
9225 undo_isolate_page_range(pfn_max_align_down(start),
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02009226 pfn_max_align_up(end), migratetype);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009227 return ret;
9228}
David Hildenbrand255f5982020-05-07 16:01:29 +02009229EXPORT_SYMBOL(alloc_contig_range);
Anshuman Khandual5e27a2d2019-11-30 17:55:06 -08009230
9231static int __alloc_contig_pages(unsigned long start_pfn,
9232 unsigned long nr_pages, gfp_t gfp_mask)
9233{
9234 unsigned long end_pfn = start_pfn + nr_pages;
9235
9236 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
9237 gfp_mask);
9238}
9239
9240static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
9241 unsigned long nr_pages)
9242{
9243 unsigned long i, end_pfn = start_pfn + nr_pages;
9244 struct page *page;
9245
9246 for (i = start_pfn; i < end_pfn; i++) {
9247 page = pfn_to_online_page(i);
9248 if (!page)
9249 return false;
9250
9251 if (page_zone(page) != z)
9252 return false;
9253
9254 if (PageReserved(page))
9255 return false;
Anshuman Khandual5e27a2d2019-11-30 17:55:06 -08009256 }
9257 return true;
9258}
9259
9260static bool zone_spans_last_pfn(const struct zone *zone,
9261 unsigned long start_pfn, unsigned long nr_pages)
9262{
9263 unsigned long last_pfn = start_pfn + nr_pages - 1;
9264
9265 return zone_spans_pfn(zone, last_pfn);
9266}
9267
9268/**
9269 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
9270 * @nr_pages: Number of contiguous pages to allocate
9271 * @gfp_mask: GFP mask to limit search and used during compaction
9272 * @nid: Target node
9273 * @nodemask: Mask for other possible nodes
9274 *
9275 * This routine is a wrapper around alloc_contig_range(). It scans over zones
9276 * on an applicable zonelist to find a contiguous pfn range which can then be
9277 * tried for allocation with alloc_contig_range(). This routine is intended
9278 * for allocation requests which can not be fulfilled with the buddy allocator.
9279 *
9280 * The allocated memory is always aligned to a page boundary. If nr_pages is a
Anshuman Khandualeaab8e72022-01-14 14:07:33 -08009281 * power of two, then allocated range is also guaranteed to be aligned to same
9282 * nr_pages (e.g. 1GB request would be aligned to 1GB).
Anshuman Khandual5e27a2d2019-11-30 17:55:06 -08009283 *
9284 * Allocated pages can be freed with free_contig_range() or by manually calling
9285 * __free_page() on each allocated page.
9286 *
9287 * Return: pointer to contiguous pages on success, or NULL if not successful.
9288 */
9289struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
9290 int nid, nodemask_t *nodemask)
9291{
9292 unsigned long ret, pfn, flags;
9293 struct zonelist *zonelist;
9294 struct zone *zone;
9295 struct zoneref *z;
9296
9297 zonelist = node_zonelist(nid, gfp_mask);
9298 for_each_zone_zonelist_nodemask(zone, z, zonelist,
9299 gfp_zone(gfp_mask), nodemask) {
9300 spin_lock_irqsave(&zone->lock, flags);
9301
9302 pfn = ALIGN(zone->zone_start_pfn, nr_pages);
9303 while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
9304 if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
9305 /*
9306 * We release the zone lock here because
9307 * alloc_contig_range() will also lock the zone
9308 * at some point. If there's an allocation
9309 * spinning on this lock, it may win the race
9310 * and cause alloc_contig_range() to fail...
9311 */
9312 spin_unlock_irqrestore(&zone->lock, flags);
9313 ret = __alloc_contig_pages(pfn, nr_pages,
9314 gfp_mask);
9315 if (!ret)
9316 return pfn_to_page(pfn);
9317 spin_lock_irqsave(&zone->lock, flags);
9318 }
9319 pfn += nr_pages;
9320 }
9321 spin_unlock_irqrestore(&zone->lock, flags);
9322 }
9323 return NULL;
9324}
Alexandre Ghiti4eb07162019-05-13 17:19:04 -07009325#endif /* CONFIG_CONTIG_ALLOC */
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009326
Minchan Kim78fa5152021-05-04 18:37:34 -07009327void free_contig_range(unsigned long pfn, unsigned long nr_pages)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009328{
Minchan Kim78fa5152021-05-04 18:37:34 -07009329 unsigned long count = 0;
Marek Szyprowskibcc2b022012-12-20 15:05:18 -08009330
9331 for (; nr_pages--; pfn++) {
9332 struct page *page = pfn_to_page(pfn);
9333
9334 count += page_count(page) != 1;
9335 __free_page(page);
9336 }
Minchan Kim78fa5152021-05-04 18:37:34 -07009337 WARN(count != 0, "%lu pages are still in use!\n", count);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009338}
David Hildenbrand255f5982020-05-07 16:01:29 +02009339EXPORT_SYMBOL(free_contig_range);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01009340
Cody P Schafer0a647f32013-07-03 15:01:33 -07009341/*
9342 * The zone indicated has a new number of managed_pages; batch sizes and percpu
Ingo Molnarf0953a12021-05-06 18:06:47 -07009343 * page high values need to be recalculated.
Cody P Schafer0a647f32013-07-03 15:01:33 -07009344 */
Mel Gorman04f8cfe2021-06-28 19:42:15 -07009345void zone_pcp_update(struct zone *zone, int cpu_online)
Jiang Liu4ed7e022012-07-31 16:43:35 -07009346{
Cody P Schaferc8e251f2013-07-03 15:01:29 -07009347 mutex_lock(&pcp_batch_high_lock);
Mel Gorman04f8cfe2021-06-28 19:42:15 -07009348 zone_set_pageset_high_and_batch(zone, cpu_online);
Cody P Schaferc8e251f2013-07-03 15:01:29 -07009349 mutex_unlock(&pcp_batch_high_lock);
Jiang Liu4ed7e022012-07-31 16:43:35 -07009350}
Jiang Liu4ed7e022012-07-31 16:43:35 -07009351
Vlastimil Babkaec6e8c7e2020-12-14 19:10:59 -08009352/*
9353 * Effectively disable pcplists for the zone by setting the high limit to 0
9354 * and draining all cpus. A concurrent page freeing on another CPU that's about
9355 * to put the page on pcplist will either finish before the drain and the page
9356 * will be drained, or observe the new high limit and skip the pcplist.
9357 *
9358 * Must be paired with a call to zone_pcp_enable().
9359 */
9360void zone_pcp_disable(struct zone *zone)
9361{
9362 mutex_lock(&pcp_batch_high_lock);
9363 __zone_set_pageset_high_and_batch(zone, 0, 1);
9364 __drain_all_pages(zone, true);
9365}
9366
9367void zone_pcp_enable(struct zone *zone)
9368{
9369 __zone_set_pageset_high_and_batch(zone, zone->pageset_high, zone->pageset_batch);
9370 mutex_unlock(&pcp_batch_high_lock);
9371}
9372
Jiang Liu340175b2012-07-31 16:43:32 -07009373void zone_pcp_reset(struct zone *zone)
9374{
Minchan Kim5a883812012-10-08 16:33:39 -07009375 int cpu;
Mel Gorman28f836b2021-06-28 19:41:38 -07009376 struct per_cpu_zonestat *pzstats;
Jiang Liu340175b2012-07-31 16:43:32 -07009377
Mel Gorman28f836b2021-06-28 19:41:38 -07009378 if (zone->per_cpu_pageset != &boot_pageset) {
Minchan Kim5a883812012-10-08 16:33:39 -07009379 for_each_online_cpu(cpu) {
Mel Gorman28f836b2021-06-28 19:41:38 -07009380 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
9381 drain_zonestat(zone, pzstats);
Minchan Kim5a883812012-10-08 16:33:39 -07009382 }
Mel Gorman28f836b2021-06-28 19:41:38 -07009383 free_percpu(zone->per_cpu_pageset);
9384 free_percpu(zone->per_cpu_zonestats);
9385 zone->per_cpu_pageset = &boot_pageset;
9386 zone->per_cpu_zonestats = &boot_zonestats;
Jiang Liu340175b2012-07-31 16:43:32 -07009387 }
Jiang Liu340175b2012-07-31 16:43:32 -07009388}
9389
Wen Congyang6dcd73d2012-12-11 16:01:01 -08009390#ifdef CONFIG_MEMORY_HOTREMOVE
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009391/*
David Hildenbrand257bea72020-10-15 20:07:59 -07009392 * All pages in the range must be in a single zone, must not contain holes,
9393 * must span full sections, and must be isolated before calling this function.
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009394 */
David Hildenbrand257bea72020-10-15 20:07:59 -07009395void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009396{
David Hildenbrand257bea72020-10-15 20:07:59 -07009397 unsigned long pfn = start_pfn;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009398 struct page *page;
9399 struct zone *zone;
David Hildenbrand0ee5f4f2019-11-30 17:54:03 -08009400 unsigned int order;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009401 unsigned long flags;
Michal Hocko5557c762019-05-13 17:21:24 -07009402
Michal Hocko2d070ea2017-07-06 15:37:56 -07009403 offline_mem_sections(pfn, end_pfn);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009404 zone = page_zone(pfn_to_page(pfn));
9405 spin_lock_irqsave(&zone->lock, flags);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009406 while (pfn < end_pfn) {
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009407 page = pfn_to_page(pfn);
Wen Congyangb023f462012-12-11 16:00:45 -08009408 /*
9409 * The HWPoisoned page may be not in buddy system, and
9410 * page_count() is not 0.
9411 */
9412 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
9413 pfn++;
Wen Congyangb023f462012-12-11 16:00:45 -08009414 continue;
9415 }
David Hildenbrandaa218792020-05-07 16:01:30 +02009416 /*
9417 * At this point all remaining PageOffline() pages have a
9418 * reference count of 0 and can simply be skipped.
9419 */
9420 if (PageOffline(page)) {
9421 BUG_ON(page_count(page));
9422 BUG_ON(PageBuddy(page));
9423 pfn++;
David Hildenbrandaa218792020-05-07 16:01:30 +02009424 continue;
9425 }
Wen Congyangb023f462012-12-11 16:00:45 -08009426
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009427 BUG_ON(page_count(page));
9428 BUG_ON(!PageBuddy(page));
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07009429 order = buddy_order(page);
Alexander Duyck6ab01362020-04-06 20:04:49 -07009430 del_page_from_free_list(page, zone, order);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07009431 pfn += (1 << order);
9432 }
9433 spin_unlock_irqrestore(&zone->lock, flags);
9434}
9435#endif
Wu Fengguang8d22ba12009-12-16 12:19:58 +01009436
Eric Dumazet8446b592021-11-05 13:40:31 -07009437/*
9438 * This function returns a stable result only if called under zone lock.
9439 */
Wu Fengguang8d22ba12009-12-16 12:19:58 +01009440bool is_free_buddy_page(struct page *page)
9441{
Wu Fengguang8d22ba12009-12-16 12:19:58 +01009442 unsigned long pfn = page_to_pfn(page);
Mel Gorman7aeb09f2014-06-04 16:10:21 -07009443 unsigned int order;
Wu Fengguang8d22ba12009-12-16 12:19:58 +01009444
Wu Fengguang8d22ba12009-12-16 12:19:58 +01009445 for (order = 0; order < MAX_ORDER; order++) {
9446 struct page *page_head = page - (pfn & ((1 << order) - 1));
9447
Eric Dumazet8446b592021-11-05 13:40:31 -07009448 if (PageBuddy(page_head) &&
9449 buddy_order_unsafe(page_head) >= order)
Wu Fengguang8d22ba12009-12-16 12:19:58 +01009450 break;
9451 }
Wu Fengguang8d22ba12009-12-16 12:19:58 +01009452
9453 return order < MAX_ORDER;
9454}
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009455
9456#ifdef CONFIG_MEMORY_FAILURE
9457/*
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009458 * Break down a higher-order page in sub-pages, and keep our target out of
9459 * buddy allocator.
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009460 */
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009461static void break_down_buddy_pages(struct zone *zone, struct page *page,
9462 struct page *target, int low, int high,
9463 int migratetype)
9464{
9465 unsigned long size = 1 << high;
9466 struct page *current_buddy, *next_page;
9467
9468 while (high > low) {
9469 high--;
9470 size >>= 1;
9471
9472 if (target >= &page[size]) {
9473 next_page = page + size;
9474 current_buddy = page;
9475 } else {
9476 next_page = page;
9477 current_buddy = page + size;
9478 }
9479
9480 if (set_page_guard(zone, current_buddy, high, migratetype))
9481 continue;
9482
9483 if (current_buddy != target) {
9484 add_to_free_list(current_buddy, zone, high, migratetype);
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07009485 set_buddy_order(current_buddy, high);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009486 page = next_page;
9487 }
9488 }
9489}
9490
9491/*
9492 * Take a page that will be marked as poisoned off the buddy allocator.
9493 */
9494bool take_page_off_buddy(struct page *page)
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009495{
9496 struct zone *zone = page_zone(page);
9497 unsigned long pfn = page_to_pfn(page);
9498 unsigned long flags;
9499 unsigned int order;
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009500 bool ret = false;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009501
9502 spin_lock_irqsave(&zone->lock, flags);
9503 for (order = 0; order < MAX_ORDER; order++) {
9504 struct page *page_head = page - (pfn & ((1 << order) - 1));
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07009505 int page_order = buddy_order(page_head);
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009506
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07009507 if (PageBuddy(page_head) && page_order >= order) {
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009508 unsigned long pfn_head = page_to_pfn(page_head);
9509 int migratetype = get_pfnblock_migratetype(page_head,
9510 pfn_head);
9511
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07009512 del_page_from_free_list(page_head, zone, page_order);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009513 break_down_buddy_pages(zone, page_head, page, 0,
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07009514 page_order, migratetype);
Naoya Horiguchibf181c52022-01-14 14:09:09 -08009515 SetPageHWPoisonTakenOff(page);
Ding Huibac9c6f2021-06-04 20:01:21 -07009516 if (!is_migrate_isolate(migratetype))
9517 __mod_zone_freepage_state(zone, -1, migratetype);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009518 ret = true;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009519 break;
9520 }
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009521 if (page_count(page_head) > 0)
9522 break;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009523 }
9524 spin_unlock_irqrestore(&zone->lock, flags);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07009525 return ret;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009526}
Naoya Horiguchibf181c52022-01-14 14:09:09 -08009527
9528/*
9529 * Cancel takeoff done by take_page_off_buddy().
9530 */
9531bool put_page_back_buddy(struct page *page)
9532{
9533 struct zone *zone = page_zone(page);
9534 unsigned long pfn = page_to_pfn(page);
9535 unsigned long flags;
9536 int migratetype = get_pfnblock_migratetype(page, pfn);
9537 bool ret = false;
9538
9539 spin_lock_irqsave(&zone->lock, flags);
9540 if (put_page_testzero(page)) {
9541 ClearPageHWPoisonTakenOff(page);
9542 __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE);
9543 if (TestClearPageHWPoison(page)) {
9544 num_poisoned_pages_dec();
9545 ret = true;
9546 }
9547 }
9548 spin_unlock_irqrestore(&zone->lock, flags);
9549
9550 return ret;
9551}
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07009552#endif
Baoquan He62b31072022-01-14 14:07:37 -08009553
9554#ifdef CONFIG_ZONE_DMA
9555bool has_managed_dma(void)
9556{
9557 struct pglist_data *pgdat;
9558
9559 for_each_online_pgdat(pgdat) {
9560 struct zone *zone = &pgdat->node_zones[ZONE_DMA];
9561
9562 if (managed_zone(zone))
9563 return true;
9564 }
9565 return false;
9566}
9567#endif /* CONFIG_ZONE_DMA */