blob: b68ee86788af91f2ed3646543eec78404a72d52e [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Yinghai Lu95f72d12010-07-12 14:36:09 +10002/*
3 * Procedures for maintaining information about logical memory blocks.
4 *
5 * Peter Bergner, IBM Corp. June 2001.
6 * Copyright (C) 2001 Peter Bergner.
Yinghai Lu95f72d12010-07-12 14:36:09 +10007 */
8
9#include <linux/kernel.h>
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -070010#include <linux/slab.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100011#include <linux/init.h>
12#include <linux/bitops.h>
Benjamin Herrenschmidt449e8df2010-07-06 15:39:07 -070013#include <linux/poison.h>
Benjamin Herrenschmidtc196f762010-07-06 15:39:16 -070014#include <linux/pfn.h>
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -070015#include <linux/debugfs.h>
Randy Dunlap514c6032018-04-05 16:25:34 -070016#include <linux/kmemleak.h>
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -070017#include <linux/seq_file.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100018#include <linux/memblock.h>
19
Christoph Hellwigc4c5ad62016-07-28 15:48:06 -070020#include <asm/sections.h>
Santosh Shilimkar26f09e92014-01-21 15:50:19 -080021#include <linux/io.h>
22
23#include "internal.h"
Tang Chen79442ed2013-11-12 15:07:59 -080024
Ard Biesheuvel8a5b403d2019-02-15 13:33:32 +010025#define INIT_MEMBLOCK_REGIONS 128
26#define INIT_PHYSMEM_REGIONS 4
27
28#ifndef INIT_MEMBLOCK_RESERVED_REGIONS
29# define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
30#endif
31
Mike Rapoport3e039c52018-06-30 17:55:05 +030032/**
33 * DOC: memblock overview
34 *
35 * Memblock is a method of managing memory regions during the early
36 * boot period when the usual kernel memory allocators are not up and
37 * running.
38 *
39 * Memblock views the system memory as collections of contiguous
40 * regions. There are several types of these collections:
41 *
42 * * ``memory`` - describes the physical memory available to the
43 * kernel; this may differ from the actual physical memory installed
44 * in the system, for instance when the memory is restricted with
45 * ``mem=`` command line parameter
46 * * ``reserved`` - describes the regions that were allocated
David Hildenbrand77649902020-07-01 16:18:29 +020047 * * ``physmem`` - describes the actual physical memory available during
48 * boot regardless of the possible restrictions and memory hot(un)plug;
49 * the ``physmem`` type is only available on some architectures.
Mike Rapoport3e039c52018-06-30 17:55:05 +030050 *
Mauro Carvalho Chehab9303c9d2020-09-25 12:01:25 +020051 * Each region is represented by struct memblock_region that
Mike Rapoport3e039c52018-06-30 17:55:05 +030052 * defines the region extents, its attributes and NUMA node id on NUMA
Mauro Carvalho Chehab1bf162e2020-09-28 15:50:33 +020053 * systems. Every memory type is described by the struct memblock_type
54 * which contains an array of memory regions along with
David Hildenbrand77649902020-07-01 16:18:29 +020055 * the allocator metadata. The "memory" and "reserved" types are nicely
Mauro Carvalho Chehab9303c9d2020-09-25 12:01:25 +020056 * wrapped with struct memblock. This structure is statically
David Hildenbrand77649902020-07-01 16:18:29 +020057 * initialized at build time. The region arrays are initially sized to
58 * %INIT_MEMBLOCK_REGIONS for "memory" and %INIT_MEMBLOCK_RESERVED_REGIONS
59 * for "reserved". The region array for "physmem" is initially sized to
60 * %INIT_PHYSMEM_REGIONS.
Cao jin6e5af9a2019-11-30 17:56:21 -080061 * The memblock_allow_resize() enables automatic resizing of the region
62 * arrays during addition of new regions. This feature should be used
63 * with care so that memory allocated for the region array will not
64 * overlap with areas that should be reserved, for example initrd.
Mike Rapoport3e039c52018-06-30 17:55:05 +030065 *
66 * The early architecture setup should tell memblock what the physical
Cao jin6e5af9a2019-11-30 17:56:21 -080067 * memory layout is by using memblock_add() or memblock_add_node()
68 * functions. The first function does not assign the region to a NUMA
69 * node and it is appropriate for UMA systems. Yet, it is possible to
70 * use it on NUMA systems as well and assign the region to a NUMA node
71 * later in the setup process using memblock_set_node(). The
72 * memblock_add_node() performs such an assignment directly.
Mike Rapoport3e039c52018-06-30 17:55:05 +030073 *
Mike Rapoporta2974132019-03-11 23:30:54 -070074 * Once memblock is setup the memory can be allocated using one of the
75 * API variants:
76 *
Cao jin6e5af9a2019-11-30 17:56:21 -080077 * * memblock_phys_alloc*() - these functions return the **physical**
78 * address of the allocated memory
79 * * memblock_alloc*() - these functions return the **virtual** address
80 * of the allocated memory.
Mike Rapoporta2974132019-03-11 23:30:54 -070081 *
Ethon Pauldf1758d2020-06-04 16:49:16 -070082 * Note, that both API variants use implicit assumptions about allowed
Mike Rapoporta2974132019-03-11 23:30:54 -070083 * memory ranges and the fallback methods. Consult the documentation
Cao jin6e5af9a2019-11-30 17:56:21 -080084 * of memblock_alloc_internal() and memblock_alloc_range_nid()
85 * functions for more elaborate description.
Mike Rapoport3e039c52018-06-30 17:55:05 +030086 *
Cao jin6e5af9a2019-11-30 17:56:21 -080087 * As the system boot progresses, the architecture specific mem_init()
88 * function frees all the memory to the buddy page allocator.
Mike Rapoport3e039c52018-06-30 17:55:05 +030089 *
Cao jin6e5af9a2019-11-30 17:56:21 -080090 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
David Hildenbrand77649902020-07-01 16:18:29 +020091 * memblock data structures (except "physmem") will be discarded after the
92 * system initialization completes.
Mike Rapoport3e039c52018-06-30 17:55:05 +030093 */
94
Mike Rapoportbda49a82018-10-30 15:09:40 -070095#ifndef CONFIG_NEED_MULTIPLE_NODES
96struct pglist_data __refdata contig_page_data;
97EXPORT_SYMBOL(contig_page_data);
98#endif
99
100unsigned long max_low_pfn;
101unsigned long min_low_pfn;
102unsigned long max_pfn;
103unsigned long long max_possible_pfn;
104
Tejun Heofe091c22011-12-08 10:22:07 -0800105static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
Ard Biesheuvel8a5b403d2019-02-15 13:33:32 +0100106static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
Philipp Hachtmann70210ed2014-01-29 18:16:01 +0100107#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
David Hildenbrand77649902020-07-01 16:18:29 +0200108static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS];
Philipp Hachtmann70210ed2014-01-29 18:16:01 +0100109#endif
Tejun Heofe091c22011-12-08 10:22:07 -0800110
111struct memblock memblock __initdata_memblock = {
112 .memory.regions = memblock_memory_init_regions,
113 .memory.cnt = 1, /* empty dummy entry */
114 .memory.max = INIT_MEMBLOCK_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800115 .memory.name = "memory",
Tejun Heofe091c22011-12-08 10:22:07 -0800116
117 .reserved.regions = memblock_reserved_init_regions,
118 .reserved.cnt = 1, /* empty dummy entry */
Ard Biesheuvel8a5b403d2019-02-15 13:33:32 +0100119 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800120 .reserved.name = "reserved",
Tejun Heofe091c22011-12-08 10:22:07 -0800121
Tang Chen79442ed2013-11-12 15:07:59 -0800122 .bottom_up = false,
Tejun Heofe091c22011-12-08 10:22:07 -0800123 .current_limit = MEMBLOCK_ALLOC_ANYWHERE,
124};
Yinghai Lu95f72d12010-07-12 14:36:09 +1000125
David Hildenbrand77649902020-07-01 16:18:29 +0200126#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
127struct memblock_type physmem = {
128 .regions = memblock_physmem_init_regions,
129 .cnt = 1, /* empty dummy entry */
130 .max = INIT_PHYSMEM_REGIONS,
131 .name = "physmem",
132};
133#endif
134
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -0700135/*
136 * keep a pointer to &memblock.memory in the text section to use it in
137 * __next_mem_range() and its helpers.
138 * For architectures that do not keep memblock data after init, this
139 * pointer will be reset to NULL at memblock_discard()
140 */
141static __refdata struct memblock_type *memblock_memory = &memblock.memory;
142
Mike Rapoportcd991db2020-10-13 16:57:49 -0700143#define for_each_memblock_type(i, memblock_type, rgn) \
144 for (i = 0, rgn = &memblock_type->regions[0]; \
145 i < memblock_type->cnt; \
146 i++, rgn = &memblock_type->regions[i])
147
Mike Rapoport87c55872020-10-13 16:57:54 -0700148#define memblock_dbg(fmt, ...) \
149 do { \
150 if (memblock_debug) \
151 pr_info(fmt, ##__VA_ARGS__); \
152 } while (0)
153
154static int memblock_debug __initdata_memblock;
Tony Lucka3f5baf2015-06-24 16:58:12 -0700155static bool system_has_some_mirror __initdata_memblock = false;
Tejun Heo1aadc052011-12-08 10:22:08 -0800156static int memblock_can_resize __initdata_memblock;
Gavin Shan181eb392012-05-29 15:06:50 -0700157static int memblock_memory_in_slab __initdata_memblock = 0;
158static int memblock_reserved_in_slab __initdata_memblock = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000159
Mike Rapoportc366ea82019-03-11 23:29:46 -0700160static enum memblock_flags __init_memblock choose_memblock_flags(void)
Tony Lucka3f5baf2015-06-24 16:58:12 -0700161{
162 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
163}
164
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800165/* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
166static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
167{
Stefan Agner1c4bc432018-06-07 17:06:15 -0700168 return *size = min(*size, PHYS_ADDR_MAX - base);
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800169}
170
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000171/*
172 * Address comparison utilities
173 */
Yinghai Lu10d06432010-07-28 15:43:02 +1000174static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1,
Benjamin Herrenschmidt2898cc42010-08-04 13:34:42 +1000175 phys_addr_t base2, phys_addr_t size2)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000176{
177 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
178}
179
Tang Chen95cf82e2015-09-08 15:02:03 -0700180bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
H Hartley Sweeten2d7d3eb2011-10-31 17:09:15 -0700181 phys_addr_t base, phys_addr_t size)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000182{
183 unsigned long i;
184
Alexander Kuleshovf14516f2016-01-14 15:20:39 -0800185 for (i = 0; i < type->cnt; i++)
186 if (memblock_addrs_overlap(base, size, type->regions[i].base,
187 type->regions[i].size))
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000188 break;
Tang Chenc5c5c9d2015-09-08 15:02:00 -0700189 return i < type->cnt;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000190}
191
Mike Rapoport47cec442018-06-30 17:55:02 +0300192/**
Tang Chen79442ed2013-11-12 15:07:59 -0800193 * __memblock_find_range_bottom_up - find free area utility in bottom-up
194 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300195 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
196 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tang Chen79442ed2013-11-12 15:07:59 -0800197 * @size: size of free area to find
198 * @align: alignment of free area to find
Grygorii Strashkob1154232014-01-21 15:50:16 -0800199 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700200 * @flags: pick from blocks based on memory attributes
Tang Chen79442ed2013-11-12 15:07:59 -0800201 *
202 * Utility called from memblock_find_in_range_node(), find free area bottom-up.
203 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300204 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800205 * Found address on success, 0 on failure.
206 */
207static phys_addr_t __init_memblock
208__memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700209 phys_addr_t size, phys_addr_t align, int nid,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300210 enum memblock_flags flags)
Tang Chen79442ed2013-11-12 15:07:59 -0800211{
212 phys_addr_t this_start, this_end, cand;
213 u64 i;
214
Tony Luckfc6daaf2015-06-24 16:58:09 -0700215 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
Tang Chen79442ed2013-11-12 15:07:59 -0800216 this_start = clamp(this_start, start, end);
217 this_end = clamp(this_end, start, end);
218
219 cand = round_up(this_start, align);
220 if (cand < this_end && this_end - cand >= size)
221 return cand;
222 }
223
224 return 0;
225}
226
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800227/**
Tang Chen14028992013-11-12 15:07:57 -0800228 * __memblock_find_range_top_down - find free area utility, in top-down
229 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300230 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
231 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tang Chen14028992013-11-12 15:07:57 -0800232 * @size: size of free area to find
233 * @align: alignment of free area to find
Grygorii Strashkob1154232014-01-21 15:50:16 -0800234 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700235 * @flags: pick from blocks based on memory attributes
Tang Chen14028992013-11-12 15:07:57 -0800236 *
237 * Utility called from memblock_find_in_range_node(), find free area top-down.
238 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300239 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800240 * Found address on success, 0 on failure.
Tang Chen14028992013-11-12 15:07:57 -0800241 */
242static phys_addr_t __init_memblock
243__memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700244 phys_addr_t size, phys_addr_t align, int nid,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300245 enum memblock_flags flags)
Tang Chen14028992013-11-12 15:07:57 -0800246{
247 phys_addr_t this_start, this_end, cand;
248 u64 i;
249
Tony Luckfc6daaf2015-06-24 16:58:09 -0700250 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
251 NULL) {
Tang Chen14028992013-11-12 15:07:57 -0800252 this_start = clamp(this_start, start, end);
253 this_end = clamp(this_end, start, end);
254
255 if (this_end < size)
256 continue;
257
258 cand = round_down(this_end - size, align);
259 if (cand >= this_start)
260 return cand;
261 }
262
263 return 0;
264}
265
266/**
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800267 * memblock_find_in_range_node - find free area in given range and node
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800268 * @size: size of free area to find
269 * @align: alignment of free area to find
Grygorii Strashko87029ee2014-01-21 15:50:14 -0800270 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300271 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
272 * %MEMBLOCK_ALLOC_ACCESSIBLE
Grygorii Strashkob1154232014-01-21 15:50:16 -0800273 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700274 * @flags: pick from blocks based on memory attributes
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800275 *
276 * Find @size free area aligned to @align in the specified range and node.
277 *
Tang Chen79442ed2013-11-12 15:07:59 -0800278 * When allocation direction is bottom-up, the @start should be greater
279 * than the end of the kernel image. Otherwise, it will be trimmed. The
280 * reason is that we want the bottom-up allocation just near the kernel
281 * image so it is highly likely that the allocated memory and the kernel
282 * will reside in the same node.
283 *
284 * If bottom-up allocation failed, will try to allocate memory top-down.
285 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300286 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800287 * Found address on success, 0 on failure.
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000288 */
Mike Rapoportc366ea82019-03-11 23:29:46 -0700289static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
Grygorii Strashko87029ee2014-01-21 15:50:14 -0800290 phys_addr_t align, phys_addr_t start,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300291 phys_addr_t end, int nid,
292 enum memblock_flags flags)
Tang Chenf7210e62013-02-22 16:33:51 -0800293{
Tang Chen0cfb8f02014-08-29 15:18:31 -0700294 phys_addr_t kernel_end, ret;
Tang Chen79442ed2013-11-12 15:07:59 -0800295
Tang Chenf7210e62013-02-22 16:33:51 -0800296 /* pump up @end */
Qian Caifed84c72018-12-28 00:36:29 -0800297 if (end == MEMBLOCK_ALLOC_ACCESSIBLE ||
298 end == MEMBLOCK_ALLOC_KASAN)
Tang Chenf7210e62013-02-22 16:33:51 -0800299 end = memblock.current_limit;
300
301 /* avoid allocating the first page */
302 start = max_t(phys_addr_t, start, PAGE_SIZE);
303 end = max(start, end);
Tang Chen79442ed2013-11-12 15:07:59 -0800304 kernel_end = __pa_symbol(_end);
305
306 /*
307 * try bottom-up allocation only when bottom-up mode
308 * is set and @end is above the kernel image.
309 */
310 if (memblock_bottom_up() && end > kernel_end) {
311 phys_addr_t bottom_up_start;
312
313 /* make sure we will allocate above the kernel */
314 bottom_up_start = max(start, kernel_end);
315
316 /* ok, try bottom-up allocation first */
317 ret = __memblock_find_range_bottom_up(bottom_up_start, end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700318 size, align, nid, flags);
Tang Chen79442ed2013-11-12 15:07:59 -0800319 if (ret)
320 return ret;
321
322 /*
323 * we always limit bottom-up allocation above the kernel,
324 * but top-down allocation doesn't have the limit, so
325 * retrying top-down allocation may succeed when bottom-up
326 * allocation failed.
327 *
328 * bottom-up allocation is expected to be fail very rarely,
329 * so we use WARN_ONCE() here to see the stack trace if
330 * fail happens.
331 */
Michal Hockoe3d301c2018-07-13 16:59:16 -0700332 WARN_ONCE(IS_ENABLED(CONFIG_MEMORY_HOTREMOVE),
333 "memblock: bottom-up allocation failed, memory hotremove may be affected\n");
Tang Chen79442ed2013-11-12 15:07:59 -0800334 }
Tang Chenf7210e62013-02-22 16:33:51 -0800335
Tony Luckfc6daaf2015-06-24 16:58:09 -0700336 return __memblock_find_range_top_down(start, end, size, align, nid,
337 flags);
Tang Chenf7210e62013-02-22 16:33:51 -0800338}
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000339
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800340/**
341 * memblock_find_in_range - find free area in given range
342 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300343 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
344 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800345 * @size: size of free area to find
346 * @align: alignment of free area to find
347 *
348 * Find @size free area aligned to @align in the specified range.
349 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300350 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800351 * Found address on success, 0 on failure.
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800352 */
353phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
354 phys_addr_t end, phys_addr_t size,
355 phys_addr_t align)
356{
Tony Lucka3f5baf2015-06-24 16:58:12 -0700357 phys_addr_t ret;
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300358 enum memblock_flags flags = choose_memblock_flags();
Tony Lucka3f5baf2015-06-24 16:58:12 -0700359
360again:
361 ret = memblock_find_in_range_node(size, align, start, end,
362 NUMA_NO_NODE, flags);
363
364 if (!ret && (flags & MEMBLOCK_MIRROR)) {
365 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
366 &size);
367 flags &= ~MEMBLOCK_MIRROR;
368 goto again;
369 }
370
371 return ret;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800372}
373
Yinghai Lu10d06432010-07-28 15:43:02 +1000374static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000375{
Tejun Heo1440c4e2011-12-08 10:22:08 -0800376 type->total_size -= type->regions[r].size;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200377 memmove(&type->regions[r], &type->regions[r + 1],
378 (type->cnt - (r + 1)) * sizeof(type->regions[r]));
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +1000379 type->cnt--;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000380
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700381 /* Special case for empty arrays */
382 if (type->cnt == 0) {
Tejun Heo1440c4e2011-12-08 10:22:08 -0800383 WARN_ON(type->total_size != 0);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700384 type->cnt = 1;
385 type->regions[0].base = 0;
386 type->regions[0].size = 0;
Tang Chen66a20752014-01-21 15:49:20 -0800387 type->regions[0].flags = 0;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200388 memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700389 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000390}
391
Mike Rapoport350e88b2019-05-13 17:22:59 -0700392#ifndef CONFIG_ARCH_KEEP_MEMBLOCK
Pavel Tatashin3010f872017-08-18 15:16:05 -0700393/**
Mike Rapoport47cec442018-06-30 17:55:02 +0300394 * memblock_discard - discard memory and reserved arrays if they were allocated
Pavel Tatashin3010f872017-08-18 15:16:05 -0700395 */
396void __init memblock_discard(void)
Yinghai Lu29f67382012-07-11 14:02:56 -0700397{
Pavel Tatashin3010f872017-08-18 15:16:05 -0700398 phys_addr_t addr, size;
Yinghai Lu29f67382012-07-11 14:02:56 -0700399
Pavel Tatashin3010f872017-08-18 15:16:05 -0700400 if (memblock.reserved.regions != memblock_reserved_init_regions) {
401 addr = __pa(memblock.reserved.regions);
402 size = PAGE_ALIGN(sizeof(struct memblock_region) *
403 memblock.reserved.max);
404 __memblock_free_late(addr, size);
405 }
Yinghai Lu29f67382012-07-11 14:02:56 -0700406
Pavel Tatashin91b540f92017-08-25 15:55:46 -0700407 if (memblock.memory.regions != memblock_memory_init_regions) {
Pavel Tatashin3010f872017-08-18 15:16:05 -0700408 addr = __pa(memblock.memory.regions);
409 size = PAGE_ALIGN(sizeof(struct memblock_region) *
410 memblock.memory.max);
411 __memblock_free_late(addr, size);
412 }
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -0700413
414 memblock_memory = NULL;
Yinghai Lu29f67382012-07-11 14:02:56 -0700415}
Philipp Hachtmann5e270e22014-01-23 15:53:11 -0800416#endif
417
Greg Pearson48c3b582012-06-20 12:53:05 -0700418/**
419 * memblock_double_array - double the size of the memblock regions array
420 * @type: memblock type of the regions array being doubled
421 * @new_area_start: starting address of memory range to avoid overlap with
422 * @new_area_size: size of memory range to avoid overlap with
423 *
424 * Double the size of the @type regions array. If memblock is being used to
425 * allocate memory for a new reserved regions array and there is a previously
Mike Rapoport47cec442018-06-30 17:55:02 +0300426 * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
Greg Pearson48c3b582012-06-20 12:53:05 -0700427 * waiting to be reserved, ensure the memory used by the new array does
428 * not overlap.
429 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300430 * Return:
Greg Pearson48c3b582012-06-20 12:53:05 -0700431 * 0 on success, -1 on failure.
432 */
433static int __init_memblock memblock_double_array(struct memblock_type *type,
434 phys_addr_t new_area_start,
435 phys_addr_t new_area_size)
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700436{
437 struct memblock_region *new_array, *old_array;
Yinghai Lu29f67382012-07-11 14:02:56 -0700438 phys_addr_t old_alloc_size, new_alloc_size;
Mike Rapoporta36aab82018-08-17 15:47:17 -0700439 phys_addr_t old_size, new_size, addr, new_end;
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700440 int use_slab = slab_is_available();
Gavin Shan181eb392012-05-29 15:06:50 -0700441 int *in_slab;
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700442
443 /* We don't allow resizing until we know about the reserved regions
444 * of memory that aren't suitable for allocation
445 */
446 if (!memblock_can_resize)
447 return -1;
448
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700449 /* Calculate new doubled size */
450 old_size = type->max * sizeof(struct memblock_region);
451 new_size = old_size << 1;
Yinghai Lu29f67382012-07-11 14:02:56 -0700452 /*
453 * We need to allocated new one align to PAGE_SIZE,
454 * so we can free them completely later.
455 */
456 old_alloc_size = PAGE_ALIGN(old_size);
457 new_alloc_size = PAGE_ALIGN(new_size);
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700458
Gavin Shan181eb392012-05-29 15:06:50 -0700459 /* Retrieve the slab flag */
460 if (type == &memblock.memory)
461 in_slab = &memblock_memory_in_slab;
462 else
463 in_slab = &memblock_reserved_in_slab;
464
Mike Rapoporta2974132019-03-11 23:30:54 -0700465 /* Try to find some space for it */
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700466 if (use_slab) {
467 new_array = kmalloc(new_size, GFP_KERNEL);
Tejun Heo1f5026a2011-07-12 09:58:09 +0200468 addr = new_array ? __pa(new_array) : 0;
Gavin Shan4e2f0772012-05-29 15:06:50 -0700469 } else {
Greg Pearson48c3b582012-06-20 12:53:05 -0700470 /* only exclude range when trying to double reserved.regions */
471 if (type != &memblock.reserved)
472 new_area_start = new_area_size = 0;
473
474 addr = memblock_find_in_range(new_area_start + new_area_size,
475 memblock.current_limit,
Yinghai Lu29f67382012-07-11 14:02:56 -0700476 new_alloc_size, PAGE_SIZE);
Greg Pearson48c3b582012-06-20 12:53:05 -0700477 if (!addr && new_area_size)
478 addr = memblock_find_in_range(0,
Andrew Mortonfd073832012-07-31 16:42:40 -0700479 min(new_area_start, memblock.current_limit),
480 new_alloc_size, PAGE_SIZE);
Greg Pearson48c3b582012-06-20 12:53:05 -0700481
Sachin Kamat15674862012-09-04 13:55:05 +0530482 new_array = addr ? __va(addr) : NULL;
Gavin Shan4e2f0772012-05-29 15:06:50 -0700483 }
Tejun Heo1f5026a2011-07-12 09:58:09 +0200484 if (!addr) {
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700485 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800486 type->name, type->max, type->max * 2);
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700487 return -1;
488 }
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700489
Mike Rapoporta36aab82018-08-17 15:47:17 -0700490 new_end = addr + new_size - 1;
491 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
492 type->name, type->max * 2, &addr, &new_end);
Yinghai Luea9e4372010-07-28 15:13:22 +1000493
Andrew Mortonfd073832012-07-31 16:42:40 -0700494 /*
495 * Found space, we now need to move the array over before we add the
496 * reserved region since it may be our reserved array itself that is
497 * full.
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700498 */
499 memcpy(new_array, type->regions, old_size);
500 memset(new_array + type->max, 0, old_size);
501 old_array = type->regions;
502 type->regions = new_array;
503 type->max <<= 1;
504
Andrew Mortonfd073832012-07-31 16:42:40 -0700505 /* Free old array. We needn't free it if the array is the static one */
Gavin Shan181eb392012-05-29 15:06:50 -0700506 if (*in_slab)
507 kfree(old_array);
508 else if (old_array != memblock_memory_init_regions &&
509 old_array != memblock_reserved_init_regions)
Yinghai Lu29f67382012-07-11 14:02:56 -0700510 memblock_free(__pa(old_array), old_alloc_size);
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700511
Andrew Mortonfd073832012-07-31 16:42:40 -0700512 /*
513 * Reserve the new array if that comes from the memblock. Otherwise, we
514 * needn't do it
Gavin Shan181eb392012-05-29 15:06:50 -0700515 */
516 if (!use_slab)
Yinghai Lu29f67382012-07-11 14:02:56 -0700517 BUG_ON(memblock_reserve(addr, new_alloc_size));
Gavin Shan181eb392012-05-29 15:06:50 -0700518
519 /* Update slab flag */
520 *in_slab = use_slab;
521
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700522 return 0;
523}
524
Tejun Heo784656f92011-07-12 11:15:55 +0200525/**
526 * memblock_merge_regions - merge neighboring compatible regions
527 * @type: memblock type to scan
528 *
529 * Scan @type and merge neighboring compatible regions.
530 */
531static void __init_memblock memblock_merge_regions(struct memblock_type *type)
532{
533 int i = 0;
534
535 /* cnt never goes below 1 */
536 while (i < type->cnt - 1) {
537 struct memblock_region *this = &type->regions[i];
538 struct memblock_region *next = &type->regions[i + 1];
539
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200540 if (this->base + this->size != next->base ||
541 memblock_get_region_node(this) !=
Tang Chen66a20752014-01-21 15:49:20 -0800542 memblock_get_region_node(next) ||
543 this->flags != next->flags) {
Tejun Heo784656f92011-07-12 11:15:55 +0200544 BUG_ON(this->base + this->size > next->base);
545 i++;
546 continue;
547 }
548
549 this->size += next->size;
Lin Fengc0232ae2013-01-11 14:31:44 -0800550 /* move forward from next + 1, index of which is i + 2 */
551 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
Tejun Heo784656f92011-07-12 11:15:55 +0200552 type->cnt--;
553 }
554}
555
556/**
557 * memblock_insert_region - insert new memblock region
Tang Chen209ff862013-04-29 15:08:41 -0700558 * @type: memblock type to insert into
559 * @idx: index for the insertion point
560 * @base: base address of the new region
561 * @size: size of the new region
562 * @nid: node id of the new region
Tang Chen66a20752014-01-21 15:49:20 -0800563 * @flags: flags of the new region
Tejun Heo784656f92011-07-12 11:15:55 +0200564 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300565 * Insert new memblock region [@base, @base + @size) into @type at @idx.
Alexander Kuleshov412d0002016-08-04 15:31:46 -0700566 * @type must already have extra room to accommodate the new region.
Tejun Heo784656f92011-07-12 11:15:55 +0200567 */
568static void __init_memblock memblock_insert_region(struct memblock_type *type,
569 int idx, phys_addr_t base,
Tang Chen66a20752014-01-21 15:49:20 -0800570 phys_addr_t size,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300571 int nid,
572 enum memblock_flags flags)
Tejun Heo784656f92011-07-12 11:15:55 +0200573{
574 struct memblock_region *rgn = &type->regions[idx];
575
576 BUG_ON(type->cnt >= type->max);
577 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
578 rgn->base = base;
579 rgn->size = size;
Tang Chen66a20752014-01-21 15:49:20 -0800580 rgn->flags = flags;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200581 memblock_set_region_node(rgn, nid);
Tejun Heo784656f92011-07-12 11:15:55 +0200582 type->cnt++;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800583 type->total_size += size;
Tejun Heo784656f92011-07-12 11:15:55 +0200584}
585
586/**
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100587 * memblock_add_range - add new memblock region
Tejun Heo784656f92011-07-12 11:15:55 +0200588 * @type: memblock type to add new region into
589 * @base: base address of the new region
590 * @size: size of the new region
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800591 * @nid: nid of the new region
Tang Chen66a20752014-01-21 15:49:20 -0800592 * @flags: flags of the new region
Tejun Heo784656f92011-07-12 11:15:55 +0200593 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300594 * Add new memblock region [@base, @base + @size) into @type. The new region
Tejun Heo784656f92011-07-12 11:15:55 +0200595 * is allowed to overlap with existing ones - overlaps don't affect already
596 * existing regions. @type is guaranteed to be minimal (all neighbouring
597 * compatible regions are merged) after the addition.
598 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300599 * Return:
Tejun Heo784656f92011-07-12 11:15:55 +0200600 * 0 on success, -errno on failure.
601 */
Anshuman Khandual02634a42020-01-30 22:14:20 -0800602static int __init_memblock memblock_add_range(struct memblock_type *type,
Tang Chen66a20752014-01-21 15:49:20 -0800603 phys_addr_t base, phys_addr_t size,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300604 int nid, enum memblock_flags flags)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000605{
Tejun Heo784656f92011-07-12 11:15:55 +0200606 bool insert = false;
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800607 phys_addr_t obase = base;
608 phys_addr_t end = base + memblock_cap_size(base, &size);
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800609 int idx, nr_new;
610 struct memblock_region *rgn;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000611
Tejun Heob3dc6272012-04-20 08:31:34 -0700612 if (!size)
613 return 0;
614
Tejun Heo784656f92011-07-12 11:15:55 +0200615 /* special case for empty array */
616 if (type->regions[0].size == 0) {
Tejun Heo1440c4e2011-12-08 10:22:08 -0800617 WARN_ON(type->cnt != 1 || type->total_size);
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +1000618 type->regions[0].base = base;
619 type->regions[0].size = size;
Tang Chen66a20752014-01-21 15:49:20 -0800620 type->regions[0].flags = flags;
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800621 memblock_set_region_node(&type->regions[0], nid);
Tejun Heo1440c4e2011-12-08 10:22:08 -0800622 type->total_size = size;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000623 return 0;
624 }
Tejun Heo784656f92011-07-12 11:15:55 +0200625repeat:
626 /*
627 * The following is executed twice. Once with %false @insert and
628 * then with %true. The first counts the number of regions needed
Alexander Kuleshov412d0002016-08-04 15:31:46 -0700629 * to accommodate the new area. The second actually inserts them.
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700630 */
Tejun Heo784656f92011-07-12 11:15:55 +0200631 base = obase;
632 nr_new = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000633
Gioh Kim66e8b432017-11-15 17:33:42 -0800634 for_each_memblock_type(idx, type, rgn) {
Tejun Heo784656f92011-07-12 11:15:55 +0200635 phys_addr_t rbase = rgn->base;
636 phys_addr_t rend = rbase + rgn->size;
637
638 if (rbase >= end)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000639 break;
Tejun Heo784656f92011-07-12 11:15:55 +0200640 if (rend <= base)
641 continue;
642 /*
643 * @rgn overlaps. If it separates the lower part of new
644 * area, insert that portion.
645 */
646 if (rbase > base) {
Mike Rapoport3f08a302020-06-03 15:57:02 -0700647#ifdef CONFIG_NEED_MULTIPLE_NODES
Wei Yangc0a29492015-09-04 15:47:38 -0700648 WARN_ON(nid != memblock_get_region_node(rgn));
649#endif
Wei Yang4fcab5f2015-09-08 14:59:53 -0700650 WARN_ON(flags != rgn->flags);
Tejun Heo784656f92011-07-12 11:15:55 +0200651 nr_new++;
652 if (insert)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800653 memblock_insert_region(type, idx++, base,
Tang Chen66a20752014-01-21 15:49:20 -0800654 rbase - base, nid,
655 flags);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000656 }
Tejun Heo784656f92011-07-12 11:15:55 +0200657 /* area below @rend is dealt with, forget about it */
658 base = min(rend, end);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000659 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000660
Tejun Heo784656f92011-07-12 11:15:55 +0200661 /* insert the remaining portion */
662 if (base < end) {
663 nr_new++;
664 if (insert)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800665 memblock_insert_region(type, idx, base, end - base,
Tang Chen66a20752014-01-21 15:49:20 -0800666 nid, flags);
Tejun Heo784656f92011-07-12 11:15:55 +0200667 }
668
nimisoloef3cc4d2016-07-26 15:24:56 -0700669 if (!nr_new)
670 return 0;
671
Tejun Heo784656f92011-07-12 11:15:55 +0200672 /*
673 * If this was the first round, resize array and repeat for actual
674 * insertions; otherwise, merge and return.
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700675 */
Tejun Heo784656f92011-07-12 11:15:55 +0200676 if (!insert) {
677 while (type->cnt + nr_new > type->max)
Greg Pearson48c3b582012-06-20 12:53:05 -0700678 if (memblock_double_array(type, obase, size) < 0)
Tejun Heo784656f92011-07-12 11:15:55 +0200679 return -ENOMEM;
680 insert = true;
681 goto repeat;
682 } else {
683 memblock_merge_regions(type);
684 return 0;
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -0700685 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000686}
687
Mike Rapoport48a833c2018-06-30 17:55:03 +0300688/**
689 * memblock_add_node - add new memblock region within a NUMA node
690 * @base: base address of the new region
691 * @size: size of the new region
692 * @nid: nid of the new region
693 *
694 * Add new memblock region [@base, @base + @size) to the "memory"
695 * type. See memblock_add_range() description for mode details
696 *
697 * Return:
698 * 0 on success, -errno on failure.
699 */
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800700int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
701 int nid)
702{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100703 return memblock_add_range(&memblock.memory, base, size, nid, 0);
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800704}
705
Mike Rapoport48a833c2018-06-30 17:55:03 +0300706/**
707 * memblock_add - add new memblock region
708 * @base: base address of the new region
709 * @size: size of the new region
710 *
711 * Add new memblock region [@base, @base + @size) to the "memory"
712 * type. See memblock_add_range() description for mode details
713 *
714 * Return:
715 * 0 on success, -errno on failure.
716 */
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700717int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
Alexander Kuleshov6a4055b2015-04-15 16:14:44 -0700718{
Miles Chen5d63f812017-02-22 15:46:42 -0800719 phys_addr_t end = base + size - 1;
720
Anshuman Khanduala090d712020-01-30 22:14:23 -0800721 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
Miles Chen5d63f812017-02-22 15:46:42 -0800722 &base, &end, (void *)_RET_IP_);
Alexander Kuleshov6a4055b2015-04-15 16:14:44 -0700723
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700724 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000725}
726
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800727/**
728 * memblock_isolate_range - isolate given range into disjoint memblocks
729 * @type: memblock type to isolate range for
730 * @base: base of range to isolate
731 * @size: size of range to isolate
732 * @start_rgn: out parameter for the start of isolated region
733 * @end_rgn: out parameter for the end of isolated region
734 *
735 * Walk @type and ensure that regions don't cross the boundaries defined by
Mike Rapoport47cec442018-06-30 17:55:02 +0300736 * [@base, @base + @size). Crossing regions are split at the boundaries,
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800737 * which may create at most two more regions. The index of the first
738 * region inside the range is returned in *@start_rgn and end in *@end_rgn.
739 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300740 * Return:
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800741 * 0 on success, -errno on failure.
742 */
743static int __init_memblock memblock_isolate_range(struct memblock_type *type,
744 phys_addr_t base, phys_addr_t size,
745 int *start_rgn, int *end_rgn)
746{
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800747 phys_addr_t end = base + memblock_cap_size(base, &size);
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800748 int idx;
749 struct memblock_region *rgn;
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800750
751 *start_rgn = *end_rgn = 0;
752
Tejun Heob3dc6272012-04-20 08:31:34 -0700753 if (!size)
754 return 0;
755
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800756 /* we'll create at most two more regions */
757 while (type->cnt + 2 > type->max)
Greg Pearson48c3b582012-06-20 12:53:05 -0700758 if (memblock_double_array(type, base, size) < 0)
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800759 return -ENOMEM;
760
Gioh Kim66e8b432017-11-15 17:33:42 -0800761 for_each_memblock_type(idx, type, rgn) {
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800762 phys_addr_t rbase = rgn->base;
763 phys_addr_t rend = rbase + rgn->size;
764
765 if (rbase >= end)
766 break;
767 if (rend <= base)
768 continue;
769
770 if (rbase < base) {
771 /*
772 * @rgn intersects from below. Split and continue
773 * to process the next region - the new top half.
774 */
775 rgn->base = base;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800776 rgn->size -= base - rbase;
777 type->total_size -= base - rbase;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800778 memblock_insert_region(type, idx, rbase, base - rbase,
Tang Chen66a20752014-01-21 15:49:20 -0800779 memblock_get_region_node(rgn),
780 rgn->flags);
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800781 } else if (rend > end) {
782 /*
783 * @rgn intersects from above. Split and redo the
784 * current region - the new bottom half.
785 */
786 rgn->base = end;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800787 rgn->size -= end - rbase;
788 type->total_size -= end - rbase;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800789 memblock_insert_region(type, idx--, rbase, end - rbase,
Tang Chen66a20752014-01-21 15:49:20 -0800790 memblock_get_region_node(rgn),
791 rgn->flags);
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800792 } else {
793 /* @rgn is fully contained, record it */
794 if (!*end_rgn)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800795 *start_rgn = idx;
796 *end_rgn = idx + 1;
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800797 }
798 }
799
800 return 0;
801}
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800802
Alexander Kuleshov35bd16a2015-11-05 18:47:00 -0800803static int __init_memblock memblock_remove_range(struct memblock_type *type,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100804 phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000805{
Tejun Heo71936182011-12-08 10:22:07 -0800806 int start_rgn, end_rgn;
807 int i, ret;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000808
Tejun Heo71936182011-12-08 10:22:07 -0800809 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
810 if (ret)
811 return ret;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000812
Tejun Heo71936182011-12-08 10:22:07 -0800813 for (i = end_rgn - 1; i >= start_rgn; i--)
814 memblock_remove_region(type, i);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700815 return 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000816}
817
Tejun Heo581adcb2011-12-08 10:22:06 -0800818int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000819{
Minchan Kim25cf23d2018-06-07 17:07:35 -0700820 phys_addr_t end = base + size - 1;
821
Anshuman Khanduala090d712020-01-30 22:14:23 -0800822 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
Minchan Kim25cf23d2018-06-07 17:07:35 -0700823 &base, &end, (void *)_RET_IP_);
824
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100825 return memblock_remove_range(&memblock.memory, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000826}
827
Mike Rapoport4d728682018-12-28 00:35:29 -0800828/**
829 * memblock_free - free boot memory block
830 * @base: phys starting address of the boot memory block
831 * @size: size of the boot memory block in bytes
832 *
833 * Free boot memory block previously allocated by memblock_alloc_xx() API.
834 * The freeing memory will not be released to the buddy allocator.
835 */
Tejun Heo581adcb2011-12-08 10:22:06 -0800836int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000837{
Miles Chen5d63f812017-02-22 15:46:42 -0800838 phys_addr_t end = base + size - 1;
839
Anshuman Khanduala090d712020-01-30 22:14:23 -0800840 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
Miles Chen5d63f812017-02-22 15:46:42 -0800841 &base, &end, (void *)_RET_IP_);
Tejun Heo24aa0782011-07-12 11:16:06 +0200842
Catalin Marinas9099dae2016-10-11 13:55:11 -0700843 kmemleak_free_part_phys(base, size);
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100844 return memblock_remove_range(&memblock.reserved, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000845}
846
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700847int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000848{
Miles Chen5d63f812017-02-22 15:46:42 -0800849 phys_addr_t end = base + size - 1;
850
Anshuman Khanduala090d712020-01-30 22:14:23 -0800851 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
Miles Chen5d63f812017-02-22 15:46:42 -0800852 &base, &end, (void *)_RET_IP_);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000853
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700854 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000855}
856
Anshuman Khandual02634a42020-01-30 22:14:20 -0800857#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
858int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
859{
860 phys_addr_t end = base + size - 1;
861
862 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
863 &base, &end, (void *)_RET_IP_);
864
David Hildenbrand77649902020-07-01 16:18:29 +0200865 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0);
Anshuman Khandual02634a42020-01-30 22:14:20 -0800866}
867#endif
868
Tejun Heo35fd0802011-07-12 11:15:59 +0200869/**
Mike Rapoport47cec442018-06-30 17:55:02 +0300870 * memblock_setclr_flag - set or clear flag for a memory region
871 * @base: base address of the region
872 * @size: size of the region
873 * @set: set or clear the flag
874 * @flag: the flag to udpate
Tang Chen66b16ed2014-01-21 15:49:23 -0800875 *
Tony Luck4308ce12014-12-12 16:54:59 -0800876 * This function isolates region [@base, @base + @size), and sets/clears flag
Tang Chen66b16ed2014-01-21 15:49:23 -0800877 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300878 * Return: 0 on success, -errno on failure.
Tang Chen66b16ed2014-01-21 15:49:23 -0800879 */
Tony Luck4308ce12014-12-12 16:54:59 -0800880static int __init_memblock memblock_setclr_flag(phys_addr_t base,
881 phys_addr_t size, int set, int flag)
Tang Chen66b16ed2014-01-21 15:49:23 -0800882{
883 struct memblock_type *type = &memblock.memory;
884 int i, ret, start_rgn, end_rgn;
885
886 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
887 if (ret)
888 return ret;
889
Mike Rapoportfe145122019-03-11 23:30:46 -0700890 for (i = start_rgn; i < end_rgn; i++) {
891 struct memblock_region *r = &type->regions[i];
892
Tony Luck4308ce12014-12-12 16:54:59 -0800893 if (set)
Mike Rapoportfe145122019-03-11 23:30:46 -0700894 r->flags |= flag;
Tony Luck4308ce12014-12-12 16:54:59 -0800895 else
Mike Rapoportfe145122019-03-11 23:30:46 -0700896 r->flags &= ~flag;
897 }
Tang Chen66b16ed2014-01-21 15:49:23 -0800898
899 memblock_merge_regions(type);
900 return 0;
901}
902
903/**
Tony Luck4308ce12014-12-12 16:54:59 -0800904 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
905 * @base: the base phys addr of the region
906 * @size: the size of the region
907 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300908 * Return: 0 on success, -errno on failure.
Tony Luck4308ce12014-12-12 16:54:59 -0800909 */
910int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
911{
912 return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
913}
914
915/**
Tang Chen66b16ed2014-01-21 15:49:23 -0800916 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
917 * @base: the base phys addr of the region
918 * @size: the size of the region
919 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300920 * Return: 0 on success, -errno on failure.
Tang Chen66b16ed2014-01-21 15:49:23 -0800921 */
922int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
923{
Tony Luck4308ce12014-12-12 16:54:59 -0800924 return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
Tang Chen66b16ed2014-01-21 15:49:23 -0800925}
926
927/**
Tony Lucka3f5baf2015-06-24 16:58:12 -0700928 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
929 * @base: the base phys addr of the region
930 * @size: the size of the region
931 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300932 * Return: 0 on success, -errno on failure.
Tony Lucka3f5baf2015-06-24 16:58:12 -0700933 */
934int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
935{
936 system_has_some_mirror = true;
937
938 return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
939}
940
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +0100941/**
942 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
943 * @base: the base phys addr of the region
944 * @size: the size of the region
945 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300946 * Return: 0 on success, -errno on failure.
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +0100947 */
948int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
949{
950 return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
951}
Tony Lucka3f5baf2015-06-24 16:58:12 -0700952
953/**
AKASHI Takahiro4c546b82017-04-03 11:23:54 +0900954 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
955 * @base: the base phys addr of the region
956 * @size: the size of the region
957 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300958 * Return: 0 on success, -errno on failure.
AKASHI Takahiro4c546b82017-04-03 11:23:54 +0900959 */
960int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
961{
962 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
963}
964
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -0700965static bool should_skip_region(struct memblock_type *type,
966 struct memblock_region *m,
967 int nid, int flags)
Mike Rapoportc9a688a2019-03-11 23:30:50 -0700968{
969 int m_nid = memblock_get_region_node(m);
970
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -0700971 /* we never skip regions when iterating memblock.reserved or physmem */
972 if (type != memblock_memory)
973 return false;
974
Mike Rapoportc9a688a2019-03-11 23:30:50 -0700975 /* only memory regions are associated with nodes, check it */
976 if (nid != NUMA_NO_NODE && nid != m_nid)
977 return true;
978
979 /* skip hotpluggable memory regions if needed */
980 if (movable_node_is_enabled() && memblock_is_hotpluggable(m))
981 return true;
982
983 /* if we want mirror memory skip non-mirror memory regions */
984 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
985 return true;
986
987 /* skip nomap memory unless we were asked for it explicitly */
988 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
989 return true;
990
991 return false;
992}
993
Robin Holt8e7a7f82015-06-30 14:56:41 -0700994/**
Mike Rapoporta2974132019-03-11 23:30:54 -0700995 * __next_mem_range - next function for for_each_free_mem_range() etc.
Tejun Heo35fd0802011-07-12 11:15:59 +0200996 * @idx: pointer to u64 loop variable
Grygorii Strashkob1154232014-01-21 15:50:16 -0800997 * @nid: node selector, %NUMA_NO_NODE for all nodes
Tony Luckfc6daaf2015-06-24 16:58:09 -0700998 * @flags: pick from blocks based on memory attributes
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100999 * @type_a: pointer to memblock_type from where the range is taken
1000 * @type_b: pointer to memblock_type which excludes memory from being taken
Wanpeng Lidad75572012-06-20 12:53:01 -07001001 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1002 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1003 * @out_nid: ptr to int for nid of the range, can be %NULL
Tejun Heo35fd0802011-07-12 11:15:59 +02001004 *
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001005 * Find the first area from *@idx which matches @nid, fill the out
Tejun Heo35fd0802011-07-12 11:15:59 +02001006 * parameters, and update *@idx for the next iteration. The lower 32bit of
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001007 * *@idx contains index into type_a and the upper 32bit indexes the
1008 * areas before each region in type_b. For example, if type_b regions
Tejun Heo35fd0802011-07-12 11:15:59 +02001009 * look like the following,
1010 *
1011 * 0:[0-16), 1:[32-48), 2:[128-130)
1012 *
1013 * The upper 32bit indexes the following regions.
1014 *
1015 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
1016 *
1017 * As both region arrays are sorted, the function advances the two indices
1018 * in lockstep and returns each intersection.
1019 */
David Hildenbrand77649902020-07-01 16:18:29 +02001020void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
1021 struct memblock_type *type_a,
1022 struct memblock_type *type_b, phys_addr_t *out_start,
1023 phys_addr_t *out_end, int *out_nid)
Tejun Heo35fd0802011-07-12 11:15:59 +02001024{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001025 int idx_a = *idx & 0xffffffff;
1026 int idx_b = *idx >> 32;
Grygorii Strashkob1154232014-01-21 15:50:16 -08001027
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001028 if (WARN_ONCE(nid == MAX_NUMNODES,
1029 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
Grygorii Strashko560dca272014-01-21 15:50:55 -08001030 nid = NUMA_NO_NODE;
Tejun Heo35fd0802011-07-12 11:15:59 +02001031
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001032 for (; idx_a < type_a->cnt; idx_a++) {
1033 struct memblock_region *m = &type_a->regions[idx_a];
1034
Tejun Heo35fd0802011-07-12 11:15:59 +02001035 phys_addr_t m_start = m->base;
1036 phys_addr_t m_end = m->base + m->size;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001037 int m_nid = memblock_get_region_node(m);
Tejun Heo35fd0802011-07-12 11:15:59 +02001038
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -07001039 if (should_skip_region(type_a, m, nid, flags))
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001040 continue;
1041
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001042 if (!type_b) {
1043 if (out_start)
1044 *out_start = m_start;
1045 if (out_end)
1046 *out_end = m_end;
1047 if (out_nid)
1048 *out_nid = m_nid;
1049 idx_a++;
1050 *idx = (u32)idx_a | (u64)idx_b << 32;
1051 return;
1052 }
Tejun Heo35fd0802011-07-12 11:15:59 +02001053
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001054 /* scan areas before each reservation */
1055 for (; idx_b < type_b->cnt + 1; idx_b++) {
1056 struct memblock_region *r;
1057 phys_addr_t r_start;
1058 phys_addr_t r_end;
1059
1060 r = &type_b->regions[idx_b];
1061 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1062 r_end = idx_b < type_b->cnt ?
Stefan Agner1c4bc432018-06-07 17:06:15 -07001063 r->base : PHYS_ADDR_MAX;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001064
1065 /*
1066 * if idx_b advanced past idx_a,
1067 * break out to advance idx_a
1068 */
Tejun Heo35fd0802011-07-12 11:15:59 +02001069 if (r_start >= m_end)
1070 break;
1071 /* if the two regions intersect, we're done */
1072 if (m_start < r_end) {
1073 if (out_start)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001074 *out_start =
1075 max(m_start, r_start);
Tejun Heo35fd0802011-07-12 11:15:59 +02001076 if (out_end)
1077 *out_end = min(m_end, r_end);
1078 if (out_nid)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001079 *out_nid = m_nid;
Tejun Heo35fd0802011-07-12 11:15:59 +02001080 /*
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001081 * The region which ends first is
1082 * advanced for the next iteration.
Tejun Heo35fd0802011-07-12 11:15:59 +02001083 */
1084 if (m_end <= r_end)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001085 idx_a++;
Tejun Heo35fd0802011-07-12 11:15:59 +02001086 else
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001087 idx_b++;
1088 *idx = (u32)idx_a | (u64)idx_b << 32;
Tejun Heo35fd0802011-07-12 11:15:59 +02001089 return;
1090 }
1091 }
1092 }
1093
1094 /* signal end of iteration */
1095 *idx = ULLONG_MAX;
1096}
1097
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001098/**
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001099 * __next_mem_range_rev - generic next function for for_each_*_range_rev()
1100 *
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001101 * @idx: pointer to u64 loop variable
Alexander Kuleshovad5ea8c2015-09-08 15:04:22 -07001102 * @nid: node selector, %NUMA_NO_NODE for all nodes
Tony Luckfc6daaf2015-06-24 16:58:09 -07001103 * @flags: pick from blocks based on memory attributes
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001104 * @type_a: pointer to memblock_type from where the range is taken
1105 * @type_b: pointer to memblock_type which excludes memory from being taken
Wanpeng Lidad75572012-06-20 12:53:01 -07001106 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1107 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1108 * @out_nid: ptr to int for nid of the range, can be %NULL
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001109 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001110 * Finds the next range from type_a which is not marked as unsuitable
1111 * in type_b.
1112 *
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001113 * Reverse of __next_mem_range().
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001114 */
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001115void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
1116 enum memblock_flags flags,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001117 struct memblock_type *type_a,
1118 struct memblock_type *type_b,
1119 phys_addr_t *out_start,
1120 phys_addr_t *out_end, int *out_nid)
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001121{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001122 int idx_a = *idx & 0xffffffff;
1123 int idx_b = *idx >> 32;
Grygorii Strashkob1154232014-01-21 15:50:16 -08001124
Grygorii Strashko560dca272014-01-21 15:50:55 -08001125 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1126 nid = NUMA_NO_NODE;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001127
1128 if (*idx == (u64)ULLONG_MAX) {
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001129 idx_a = type_a->cnt - 1;
zijun_hue47608a2016-08-04 15:32:00 -07001130 if (type_b != NULL)
1131 idx_b = type_b->cnt;
1132 else
1133 idx_b = 0;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001134 }
1135
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001136 for (; idx_a >= 0; idx_a--) {
1137 struct memblock_region *m = &type_a->regions[idx_a];
1138
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001139 phys_addr_t m_start = m->base;
1140 phys_addr_t m_end = m->base + m->size;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001141 int m_nid = memblock_get_region_node(m);
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001142
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -07001143 if (should_skip_region(type_a, m, nid, flags))
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001144 continue;
1145
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001146 if (!type_b) {
1147 if (out_start)
1148 *out_start = m_start;
1149 if (out_end)
1150 *out_end = m_end;
1151 if (out_nid)
1152 *out_nid = m_nid;
zijun_hufb399b42016-07-28 15:48:56 -07001153 idx_a--;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001154 *idx = (u32)idx_a | (u64)idx_b << 32;
1155 return;
1156 }
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001157
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001158 /* scan areas before each reservation */
1159 for (; idx_b >= 0; idx_b--) {
1160 struct memblock_region *r;
1161 phys_addr_t r_start;
1162 phys_addr_t r_end;
1163
1164 r = &type_b->regions[idx_b];
1165 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1166 r_end = idx_b < type_b->cnt ?
Stefan Agner1c4bc432018-06-07 17:06:15 -07001167 r->base : PHYS_ADDR_MAX;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001168 /*
1169 * if idx_b advanced past idx_a,
1170 * break out to advance idx_a
1171 */
1172
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001173 if (r_end <= m_start)
1174 break;
1175 /* if the two regions intersect, we're done */
1176 if (m_end > r_start) {
1177 if (out_start)
1178 *out_start = max(m_start, r_start);
1179 if (out_end)
1180 *out_end = min(m_end, r_end);
1181 if (out_nid)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001182 *out_nid = m_nid;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001183 if (m_start >= r_start)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001184 idx_a--;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001185 else
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001186 idx_b--;
1187 *idx = (u32)idx_a | (u64)idx_b << 32;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001188 return;
1189 }
1190 }
1191 }
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001192 /* signal end of iteration */
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001193 *idx = ULLONG_MAX;
1194}
1195
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001196/*
Chen Chang45e79812018-11-16 15:08:57 -08001197 * Common iterator interface used to define for_each_mem_pfn_range().
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001198 */
1199void __init_memblock __next_mem_pfn_range(int *idx, int nid,
1200 unsigned long *out_start_pfn,
1201 unsigned long *out_end_pfn, int *out_nid)
1202{
1203 struct memblock_type *type = &memblock.memory;
1204 struct memblock_region *r;
Mike Rapoportd622abf2020-06-03 15:56:53 -07001205 int r_nid;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001206
1207 while (++*idx < type->cnt) {
1208 r = &type->regions[*idx];
Mike Rapoportd622abf2020-06-03 15:56:53 -07001209 r_nid = memblock_get_region_node(r);
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001210
1211 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
1212 continue;
Mike Rapoportd622abf2020-06-03 15:56:53 -07001213 if (nid == MAX_NUMNODES || nid == r_nid)
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001214 break;
1215 }
1216 if (*idx >= type->cnt) {
1217 *idx = -1;
1218 return;
1219 }
1220
1221 if (out_start_pfn)
1222 *out_start_pfn = PFN_UP(r->base);
1223 if (out_end_pfn)
1224 *out_end_pfn = PFN_DOWN(r->base + r->size);
1225 if (out_nid)
Mike Rapoportd622abf2020-06-03 15:56:53 -07001226 *out_nid = r_nid;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001227}
1228
1229/**
1230 * memblock_set_node - set node ID on memblock regions
1231 * @base: base of area to set node ID for
1232 * @size: size of area to set node ID for
Tang Chene7e8de52014-01-21 15:49:26 -08001233 * @type: memblock type to set node ID for
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001234 * @nid: node ID to set
1235 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001236 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001237 * Regions which cross the area boundaries are split as necessary.
1238 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001239 * Return:
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001240 * 0 on success, -errno on failure.
1241 */
1242int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
Tang Chene7e8de52014-01-21 15:49:26 -08001243 struct memblock_type *type, int nid)
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001244{
Mike Rapoport3f08a302020-06-03 15:57:02 -07001245#ifdef CONFIG_NEED_MULTIPLE_NODES
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001246 int start_rgn, end_rgn;
1247 int i, ret;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001248
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001249 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
1250 if (ret)
1251 return ret;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001252
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001253 for (i = start_rgn; i < end_rgn; i++)
Wanpeng Lie9d24ad2012-10-08 16:32:21 -07001254 memblock_set_region_node(&type->regions[i], nid);
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001255
1256 memblock_merge_regions(type);
Mike Rapoport3f08a302020-06-03 15:57:02 -07001257#endif
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001258 return 0;
1259}
Mike Rapoport3f08a302020-06-03 15:57:02 -07001260
Alexander Duyck837566e2019-05-13 17:21:17 -07001261#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1262/**
1263 * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
1264 *
1265 * @idx: pointer to u64 loop variable
1266 * @zone: zone in which all of the memory blocks reside
1267 * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
1268 * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
1269 *
1270 * This function is meant to be a zone/pfn specific wrapper for the
1271 * for_each_mem_range type iterators. Specifically they are used in the
1272 * deferred memory init routines and as such we were duplicating much of
1273 * this logic throughout the code. So instead of having it in multiple
1274 * locations it seemed like it would make more sense to centralize this to
1275 * one new iterator that does everything they need.
1276 */
1277void __init_memblock
1278__next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
1279 unsigned long *out_spfn, unsigned long *out_epfn)
1280{
1281 int zone_nid = zone_to_nid(zone);
1282 phys_addr_t spa, epa;
1283 int nid;
1284
1285 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1286 &memblock.memory, &memblock.reserved,
1287 &spa, &epa, &nid);
1288
1289 while (*idx != U64_MAX) {
1290 unsigned long epfn = PFN_DOWN(epa);
1291 unsigned long spfn = PFN_UP(spa);
1292
1293 /*
1294 * Verify the end is at least past the start of the zone and
1295 * that we have at least one PFN to initialize.
1296 */
1297 if (zone->zone_start_pfn < epfn && spfn < epfn) {
1298 /* if we went too far just stop searching */
1299 if (zone_end_pfn(zone) <= spfn) {
1300 *idx = U64_MAX;
1301 break;
1302 }
1303
1304 if (out_spfn)
1305 *out_spfn = max(zone->zone_start_pfn, spfn);
1306 if (out_epfn)
1307 *out_epfn = min(zone_end_pfn(zone), epfn);
1308
1309 return;
1310 }
1311
1312 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1313 &memblock.memory, &memblock.reserved,
1314 &spa, &epa, &nid);
1315 }
1316
1317 /* signal end of iteration */
1318 if (out_spfn)
1319 *out_spfn = ULONG_MAX;
1320 if (out_epfn)
1321 *out_epfn = 0;
1322}
1323
1324#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001325
Mike Rapoport92d12f92019-03-11 23:29:41 -07001326/**
1327 * memblock_alloc_range_nid - allocate boot memory block
1328 * @size: size of memory block to be allocated in bytes
1329 * @align: alignment of the region and block's size
1330 * @start: the lower bound of the memory region to allocate (phys address)
1331 * @end: the upper bound of the memory region to allocate (phys address)
1332 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001333 * @exact_nid: control the allocation fall back to other nodes
Mike Rapoport92d12f92019-03-11 23:29:41 -07001334 *
1335 * The allocation is performed from memory region limited by
Cao jin95830662019-11-30 17:56:24 -08001336 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
Mike Rapoport92d12f92019-03-11 23:29:41 -07001337 *
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001338 * If the specified node can not hold the requested memory and @exact_nid
1339 * is false, the allocation falls back to any node in the system.
Mike Rapoport92d12f92019-03-11 23:29:41 -07001340 *
1341 * For systems with memory mirroring, the allocation is attempted first
1342 * from the regions with mirroring enabled and then retried from any
1343 * memory region.
1344 *
1345 * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for
1346 * allocated boot memory block, so that it is never reported as leaks.
1347 *
1348 * Return:
1349 * Physical address of allocated memory block on success, %0 on failure.
1350 */
Aslan Bakirov8676af12020-04-10 14:32:42 -07001351phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001352 phys_addr_t align, phys_addr_t start,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001353 phys_addr_t end, int nid,
1354 bool exact_nid)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001355{
Mike Rapoport92d12f92019-03-11 23:29:41 -07001356 enum memblock_flags flags = choose_memblock_flags();
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001357 phys_addr_t found;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001358
Mike Rapoport92d12f92019-03-11 23:29:41 -07001359 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1360 nid = NUMA_NO_NODE;
1361
Mike Rapoport2f770802018-10-30 15:10:01 -07001362 if (!align) {
1363 /* Can't use WARNs this early in boot on powerpc */
1364 dump_stack();
1365 align = SMP_CACHE_BYTES;
1366 }
1367
Mike Rapoport92d12f92019-03-11 23:29:41 -07001368again:
Tony Luckfc6daaf2015-06-24 16:58:09 -07001369 found = memblock_find_in_range_node(size, align, start, end, nid,
1370 flags);
Mike Rapoport92d12f92019-03-11 23:29:41 -07001371 if (found && !memblock_reserve(found, size))
1372 goto done;
1373
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001374 if (nid != NUMA_NO_NODE && !exact_nid) {
Mike Rapoport92d12f92019-03-11 23:29:41 -07001375 found = memblock_find_in_range_node(size, align, start,
1376 end, NUMA_NO_NODE,
1377 flags);
1378 if (found && !memblock_reserve(found, size))
1379 goto done;
1380 }
1381
1382 if (flags & MEMBLOCK_MIRROR) {
1383 flags &= ~MEMBLOCK_MIRROR;
1384 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
1385 &size);
1386 goto again;
1387 }
1388
1389 return 0;
1390
1391done:
1392 /* Skip kmemleak for kasan_init() due to high volume. */
1393 if (end != MEMBLOCK_ALLOC_KASAN)
Catalin Marinasaedf95e2014-06-06 14:38:20 -07001394 /*
Mike Rapoport92d12f92019-03-11 23:29:41 -07001395 * The min_count is set to 0 so that memblock allocated
1396 * blocks are never reported as leaks. This is because many
1397 * of these blocks are only referred via the physical
1398 * address which is not looked up by kmemleak.
Catalin Marinasaedf95e2014-06-06 14:38:20 -07001399 */
Catalin Marinas9099dae2016-10-11 13:55:11 -07001400 kmemleak_alloc_phys(found, size, 0, 0);
Mike Rapoport92d12f92019-03-11 23:29:41 -07001401
1402 return found;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001403}
1404
Mike Rapoporta2974132019-03-11 23:30:54 -07001405/**
1406 * memblock_phys_alloc_range - allocate a memory block inside specified range
1407 * @size: size of memory block to be allocated in bytes
1408 * @align: alignment of the region and block's size
1409 * @start: the lower bound of the memory region to allocate (physical address)
1410 * @end: the upper bound of the memory region to allocate (physical address)
1411 *
1412 * Allocate @size bytes in the between @start and @end.
1413 *
1414 * Return: physical address of the allocated memory block on success,
1415 * %0 on failure.
1416 */
Mike Rapoport8a770c22019-03-11 23:29:16 -07001417phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
1418 phys_addr_t align,
1419 phys_addr_t start,
1420 phys_addr_t end)
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001421{
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001422 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1423 false);
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001424}
1425
Mike Rapoporta2974132019-03-11 23:30:54 -07001426/**
1427 * memblock_phys_alloc_try_nid - allocate a memory block from specified MUMA node
1428 * @size: size of memory block to be allocated in bytes
1429 * @align: alignment of the region and block's size
1430 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1431 *
1432 * Allocates memory block from the specified NUMA node. If the node
1433 * has no available memory, attempts to allocated from any node in the
1434 * system.
1435 *
1436 * Return: physical address of the allocated memory block on success,
1437 * %0 on failure.
1438 */
Mike Rapoport9a8dd702018-10-30 15:07:59 -07001439phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
Benjamin Herrenschmidt9d1e2492010-07-06 15:39:17 -07001440{
Mike Rapoport33755572019-03-11 23:29:21 -07001441 return memblock_alloc_range_nid(size, align, 0,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001442 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001443}
1444
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001445/**
Mike Rapoporteb31d552018-10-30 15:08:04 -07001446 * memblock_alloc_internal - allocate boot memory block
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001447 * @size: size of memory block to be allocated in bytes
1448 * @align: alignment of the region and block's size
1449 * @min_addr: the lower bound of the memory region to allocate (phys address)
1450 * @max_addr: the upper bound of the memory region to allocate (phys address)
1451 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001452 * @exact_nid: control the allocation fall back to other nodes
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001453 *
Mike Rapoport92d12f92019-03-11 23:29:41 -07001454 * Allocates memory block using memblock_alloc_range_nid() and
1455 * converts the returned physical address to virtual.
1456 *
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001457 * The @min_addr limit is dropped if it can not be satisfied and the allocation
Mike Rapoport92d12f92019-03-11 23:29:41 -07001458 * will fall back to memory below @min_addr. Other constraints, such
1459 * as node and mirrored memory will be handled again in
1460 * memblock_alloc_range_nid().
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001461 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001462 * Return:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001463 * Virtual address of allocated memory block on success, NULL on failure.
1464 */
Mike Rapoporteb31d552018-10-30 15:08:04 -07001465static void * __init memblock_alloc_internal(
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001466 phys_addr_t size, phys_addr_t align,
1467 phys_addr_t min_addr, phys_addr_t max_addr,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001468 int nid, bool exact_nid)
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001469{
1470 phys_addr_t alloc;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001471
1472 /*
1473 * Detect any accidental use of these APIs after slab is ready, as at
1474 * this moment memblock may be deinitialized already and its
Mike Rapoportc6ffc5c2018-10-30 15:09:30 -07001475 * internal data may be destroyed (after execution of memblock_free_all)
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001476 */
1477 if (WARN_ON_ONCE(slab_is_available()))
1478 return kzalloc_node(size, GFP_NOWAIT, nid);
1479
Mike Rapoportf3057ad2019-10-18 20:20:01 -07001480 if (max_addr > memblock.current_limit)
1481 max_addr = memblock.current_limit;
1482
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001483 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
1484 exact_nid);
Mike Rapoport2f770802018-10-30 15:10:01 -07001485
Mike Rapoport92d12f92019-03-11 23:29:41 -07001486 /* retry allocation without lower limit */
1487 if (!alloc && min_addr)
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001488 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
1489 exact_nid);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001490
Mike Rapoport92d12f92019-03-11 23:29:41 -07001491 if (!alloc)
1492 return NULL;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001493
Mike Rapoport92d12f92019-03-11 23:29:41 -07001494 return phys_to_virt(alloc);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001495}
1496
1497/**
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001498 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
1499 * without zeroing memory
1500 * @size: size of memory block to be allocated in bytes
1501 * @align: alignment of the region and block's size
1502 * @min_addr: the lower bound of the memory region from where the allocation
1503 * is preferred (phys address)
1504 * @max_addr: the upper bound of the memory region from where the allocation
1505 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1506 * allocate only from memory limited by memblock.current_limit value
1507 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1508 *
1509 * Public function, provides additional debug information (including caller
1510 * info), if enabled. Does not zero allocated memory.
1511 *
1512 * Return:
1513 * Virtual address of allocated memory block on success, NULL on failure.
1514 */
1515void * __init memblock_alloc_exact_nid_raw(
1516 phys_addr_t size, phys_addr_t align,
1517 phys_addr_t min_addr, phys_addr_t max_addr,
1518 int nid)
1519{
1520 void *ptr;
1521
1522 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1523 __func__, (u64)size, (u64)align, nid, &min_addr,
1524 &max_addr, (void *)_RET_IP_);
1525
1526 ptr = memblock_alloc_internal(size, align,
1527 min_addr, max_addr, nid, true);
1528 if (ptr && size > 0)
1529 page_init_poison(ptr, size);
1530
1531 return ptr;
1532}
1533
1534/**
Mike Rapoporteb31d552018-10-30 15:08:04 -07001535 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001536 * memory and without panicking
1537 * @size: size of memory block to be allocated in bytes
1538 * @align: alignment of the region and block's size
1539 * @min_addr: the lower bound of the memory region from where the allocation
1540 * is preferred (phys address)
1541 * @max_addr: the upper bound of the memory region from where the allocation
Mike Rapoport97ad1082018-10-30 15:09:44 -07001542 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001543 * allocate only from memory limited by memblock.current_limit value
1544 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1545 *
1546 * Public function, provides additional debug information (including caller
1547 * info), if enabled. Does not zero allocated memory, does not panic if request
1548 * cannot be satisfied.
1549 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001550 * Return:
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001551 * Virtual address of allocated memory block on success, NULL on failure.
1552 */
Mike Rapoporteb31d552018-10-30 15:08:04 -07001553void * __init memblock_alloc_try_nid_raw(
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001554 phys_addr_t size, phys_addr_t align,
1555 phys_addr_t min_addr, phys_addr_t max_addr,
1556 int nid)
1557{
1558 void *ptr;
1559
Sakari Ailusd75f7732019-03-25 21:32:28 +02001560 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
Mike Rapoporta36aab82018-08-17 15:47:17 -07001561 __func__, (u64)size, (u64)align, nid, &min_addr,
1562 &max_addr, (void *)_RET_IP_);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001563
Mike Rapoporteb31d552018-10-30 15:08:04 -07001564 ptr = memblock_alloc_internal(size, align,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001565 min_addr, max_addr, nid, false);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001566 if (ptr && size > 0)
Alexander Duyckf682a972018-10-26 15:07:45 -07001567 page_init_poison(ptr, size);
1568
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001569 return ptr;
1570}
1571
1572/**
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001573 * memblock_alloc_try_nid - allocate boot memory block
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001574 * @size: size of memory block to be allocated in bytes
1575 * @align: alignment of the region and block's size
1576 * @min_addr: the lower bound of the memory region from where the allocation
1577 * is preferred (phys address)
1578 * @max_addr: the upper bound of the memory region from where the allocation
Mike Rapoport97ad1082018-10-30 15:09:44 -07001579 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001580 * allocate only from memory limited by memblock.current_limit value
1581 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1582 *
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001583 * Public function, provides additional debug information (including caller
1584 * info), if enabled. This function zeroes the allocated memory.
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001585 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001586 * Return:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001587 * Virtual address of allocated memory block on success, NULL on failure.
1588 */
Mike Rapoporteb31d552018-10-30 15:08:04 -07001589void * __init memblock_alloc_try_nid(
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001590 phys_addr_t size, phys_addr_t align,
1591 phys_addr_t min_addr, phys_addr_t max_addr,
1592 int nid)
1593{
1594 void *ptr;
1595
Sakari Ailusd75f7732019-03-25 21:32:28 +02001596 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
Mike Rapoporta36aab82018-08-17 15:47:17 -07001597 __func__, (u64)size, (u64)align, nid, &min_addr,
1598 &max_addr, (void *)_RET_IP_);
Mike Rapoporteb31d552018-10-30 15:08:04 -07001599 ptr = memblock_alloc_internal(size, align,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001600 min_addr, max_addr, nid, false);
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001601 if (ptr)
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001602 memset(ptr, 0, size);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001603
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001604 return ptr;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001605}
1606
1607/**
Mike Rapoporta2974132019-03-11 23:30:54 -07001608 * __memblock_free_late - free pages directly to buddy allocator
Mike Rapoport48a833c2018-06-30 17:55:03 +03001609 * @base: phys starting address of the boot memory block
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001610 * @size: size of the boot memory block in bytes
1611 *
Mike Rapoporta2974132019-03-11 23:30:54 -07001612 * This is only useful when the memblock allocator has already been torn
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001613 * down, but we are still initializing the system. Pages are released directly
Mike Rapoporta2974132019-03-11 23:30:54 -07001614 * to the buddy allocator.
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001615 */
1616void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)
1617{
Mike Rapoporta36aab82018-08-17 15:47:17 -07001618 phys_addr_t cursor, end;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001619
Mike Rapoporta36aab82018-08-17 15:47:17 -07001620 end = base + size - 1;
Sakari Ailusd75f7732019-03-25 21:32:28 +02001621 memblock_dbg("%s: [%pa-%pa] %pS\n",
Mike Rapoporta36aab82018-08-17 15:47:17 -07001622 __func__, &base, &end, (void *)_RET_IP_);
Catalin Marinas9099dae2016-10-11 13:55:11 -07001623 kmemleak_free_part_phys(base, size);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001624 cursor = PFN_UP(base);
1625 end = PFN_DOWN(base + size);
1626
1627 for (; cursor < end; cursor++) {
Mike Rapoport7c2ee342018-10-30 15:09:36 -07001628 memblock_free_pages(pfn_to_page(cursor), cursor, 0);
Arun KSca79b0c2018-12-28 00:34:29 -08001629 totalram_pages_inc();
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001630 }
1631}
Benjamin Herrenschmidt9d1e2492010-07-06 15:39:17 -07001632
1633/*
1634 * Remaining API functions
1635 */
1636
David Gibson1f1ffb8a2016-02-05 15:36:19 -08001637phys_addr_t __init_memblock memblock_phys_mem_size(void)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001638{
Tejun Heo1440c4e2011-12-08 10:22:08 -08001639 return memblock.memory.total_size;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001640}
1641
Srikar Dronamraju8907de52016-10-07 16:59:18 -07001642phys_addr_t __init_memblock memblock_reserved_size(void)
1643{
1644 return memblock.reserved.total_size;
1645}
1646
Sam Ravnborg0a93ebe2011-10-31 17:08:16 -07001647/* lowest address */
1648phys_addr_t __init_memblock memblock_start_of_DRAM(void)
1649{
1650 return memblock.memory.regions[0].base;
1651}
1652
Yinghai Lu10d06432010-07-28 15:43:02 +10001653phys_addr_t __init_memblock memblock_end_of_DRAM(void)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001654{
1655 int idx = memblock.memory.cnt - 1;
1656
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +10001657 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001658}
1659
Dennis Chena571d4e2016-07-28 15:48:26 -07001660static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001661{
Stefan Agner1c4bc432018-06-07 17:06:15 -07001662 phys_addr_t max_addr = PHYS_ADDR_MAX;
Emil Medve136199f2014-04-07 15:37:52 -07001663 struct memblock_region *r;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001664
Dennis Chena571d4e2016-07-28 15:48:26 -07001665 /*
1666 * translate the memory @limit size into the max address within one of
1667 * the memory memblock regions, if the @limit exceeds the total size
Stefan Agner1c4bc432018-06-07 17:06:15 -07001668 * of those regions, max_addr will keep original value PHYS_ADDR_MAX
Dennis Chena571d4e2016-07-28 15:48:26 -07001669 */
Mike Rapoportcc6de162020-10-13 16:58:30 -07001670 for_each_mem_region(r) {
Tejun Heoc0ce8fe2011-12-08 10:22:07 -08001671 if (limit <= r->size) {
1672 max_addr = r->base + limit;
1673 break;
1674 }
1675 limit -= r->size;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001676 }
1677
Dennis Chena571d4e2016-07-28 15:48:26 -07001678 return max_addr;
1679}
1680
1681void __init memblock_enforce_memory_limit(phys_addr_t limit)
1682{
Colin Ian King49aef712020-04-01 21:11:01 -07001683 phys_addr_t max_addr;
Dennis Chena571d4e2016-07-28 15:48:26 -07001684
1685 if (!limit)
1686 return;
1687
1688 max_addr = __find_max_addr(limit);
1689
1690 /* @limit exceeds the total size of the memory, do nothing */
Stefan Agner1c4bc432018-06-07 17:06:15 -07001691 if (max_addr == PHYS_ADDR_MAX)
Dennis Chena571d4e2016-07-28 15:48:26 -07001692 return;
1693
Tejun Heoc0ce8fe2011-12-08 10:22:07 -08001694 /* truncate both memory and reserved regions */
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001695 memblock_remove_range(&memblock.memory, max_addr,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001696 PHYS_ADDR_MAX);
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001697 memblock_remove_range(&memblock.reserved, max_addr,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001698 PHYS_ADDR_MAX);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001699}
1700
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001701void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
1702{
1703 int start_rgn, end_rgn;
1704 int i, ret;
1705
1706 if (!size)
1707 return;
1708
1709 ret = memblock_isolate_range(&memblock.memory, base, size,
1710 &start_rgn, &end_rgn);
1711 if (ret)
1712 return;
1713
1714 /* remove all the MAP regions */
1715 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
1716 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1717 memblock_remove_region(&memblock.memory, i);
1718
1719 for (i = start_rgn - 1; i >= 0; i--)
1720 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1721 memblock_remove_region(&memblock.memory, i);
1722
1723 /* truncate the reserved regions */
1724 memblock_remove_range(&memblock.reserved, 0, base);
1725 memblock_remove_range(&memblock.reserved,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001726 base + size, PHYS_ADDR_MAX);
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001727}
1728
Dennis Chena571d4e2016-07-28 15:48:26 -07001729void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1730{
Dennis Chena571d4e2016-07-28 15:48:26 -07001731 phys_addr_t max_addr;
Dennis Chena571d4e2016-07-28 15:48:26 -07001732
1733 if (!limit)
1734 return;
1735
1736 max_addr = __find_max_addr(limit);
1737
1738 /* @limit exceeds the total size of the memory, do nothing */
Stefan Agner1c4bc432018-06-07 17:06:15 -07001739 if (max_addr == PHYS_ADDR_MAX)
Dennis Chena571d4e2016-07-28 15:48:26 -07001740 return;
1741
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001742 memblock_cap_memory_range(0, max_addr);
Dennis Chena571d4e2016-07-28 15:48:26 -07001743}
1744
Yinghai Lucd794812010-10-11 12:34:09 -07001745static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001746{
1747 unsigned int left = 0, right = type->cnt;
1748
1749 do {
1750 unsigned int mid = (right + left) / 2;
1751
1752 if (addr < type->regions[mid].base)
1753 right = mid;
1754 else if (addr >= (type->regions[mid].base +
1755 type->regions[mid].size))
1756 left = mid + 1;
1757 else
1758 return mid;
1759 } while (left < right);
1760 return -1;
1761}
1762
Yueyi Lif5a222d2018-12-14 14:17:06 -08001763bool __init_memblock memblock_is_reserved(phys_addr_t addr)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001764{
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001765 return memblock_search(&memblock.reserved, addr) != -1;
1766}
Yinghai Lu95f72d12010-07-12 14:36:09 +10001767
Yaowei Baib4ad0c72016-01-14 15:18:54 -08001768bool __init_memblock memblock_is_memory(phys_addr_t addr)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001769{
1770 return memblock_search(&memblock.memory, addr) != -1;
1771}
1772
Yaowei Bai937f0c22018-02-06 15:41:18 -08001773bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001774{
1775 int i = memblock_search(&memblock.memory, addr);
1776
1777 if (i == -1)
1778 return false;
1779 return !memblock_is_nomap(&memblock.memory.regions[i]);
1780}
1781
Yinghai Lue76b63f2013-09-11 14:22:17 -07001782int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
1783 unsigned long *start_pfn, unsigned long *end_pfn)
1784{
1785 struct memblock_type *type = &memblock.memory;
Fabian Frederick16763232014-04-07 15:37:53 -07001786 int mid = memblock_search(type, PFN_PHYS(pfn));
Yinghai Lue76b63f2013-09-11 14:22:17 -07001787
1788 if (mid == -1)
1789 return -1;
1790
Fabian Frederickf7e2f7e2014-06-04 16:07:51 -07001791 *start_pfn = PFN_DOWN(type->regions[mid].base);
1792 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
Yinghai Lue76b63f2013-09-11 14:22:17 -07001793
Mike Rapoportd622abf2020-06-03 15:56:53 -07001794 return memblock_get_region_node(&type->regions[mid]);
Yinghai Lue76b63f2013-09-11 14:22:17 -07001795}
Yinghai Lue76b63f2013-09-11 14:22:17 -07001796
Stephen Boydeab30942012-05-24 00:45:21 -07001797/**
1798 * memblock_is_region_memory - check if a region is a subset of memory
1799 * @base: base of region to check
1800 * @size: size of region to check
1801 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001802 * Check if the region [@base, @base + @size) is a subset of a memory block.
Stephen Boydeab30942012-05-24 00:45:21 -07001803 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001804 * Return:
Stephen Boydeab30942012-05-24 00:45:21 -07001805 * 0 if false, non-zero if true
1806 */
Yaowei Bai937f0c22018-02-06 15:41:18 -08001807bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001808{
Tomi Valkeinenabb65272011-01-20 14:44:20 -08001809 int idx = memblock_search(&memblock.memory, base);
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001810 phys_addr_t end = base + memblock_cap_size(base, &size);
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001811
1812 if (idx == -1)
Yaowei Bai937f0c22018-02-06 15:41:18 -08001813 return false;
Wei Yangef415ef2017-02-22 15:45:04 -08001814 return (memblock.memory.regions[idx].base +
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001815 memblock.memory.regions[idx].size) >= end;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001816}
1817
Stephen Boydeab30942012-05-24 00:45:21 -07001818/**
1819 * memblock_is_region_reserved - check if a region intersects reserved memory
1820 * @base: base of region to check
1821 * @size: size of region to check
1822 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001823 * Check if the region [@base, @base + @size) intersects a reserved
1824 * memory block.
Stephen Boydeab30942012-05-24 00:45:21 -07001825 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001826 * Return:
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001827 * True if they intersect, false if not.
Stephen Boydeab30942012-05-24 00:45:21 -07001828 */
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001829bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001830{
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001831 memblock_cap_size(base, &size);
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001832 return memblock_overlaps_region(&memblock.reserved, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001833}
1834
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001835void __init_memblock memblock_trim_memory(phys_addr_t align)
1836{
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001837 phys_addr_t start, end, orig_start, orig_end;
Emil Medve136199f2014-04-07 15:37:52 -07001838 struct memblock_region *r;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001839
Mike Rapoportcc6de162020-10-13 16:58:30 -07001840 for_each_mem_region(r) {
Emil Medve136199f2014-04-07 15:37:52 -07001841 orig_start = r->base;
1842 orig_end = r->base + r->size;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001843 start = round_up(orig_start, align);
1844 end = round_down(orig_end, align);
1845
1846 if (start == orig_start && end == orig_end)
1847 continue;
1848
1849 if (start < end) {
Emil Medve136199f2014-04-07 15:37:52 -07001850 r->base = start;
1851 r->size = end - start;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001852 } else {
Emil Medve136199f2014-04-07 15:37:52 -07001853 memblock_remove_region(&memblock.memory,
1854 r - memblock.memory.regions);
1855 r--;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001856 }
1857 }
1858}
Benjamin Herrenschmidte63075a2010-07-06 15:39:01 -07001859
Yinghai Lu3661ca62010-09-15 13:05:29 -07001860void __init_memblock memblock_set_current_limit(phys_addr_t limit)
Benjamin Herrenschmidte63075a2010-07-06 15:39:01 -07001861{
1862 memblock.current_limit = limit;
1863}
1864
Laura Abbottfec51012014-02-27 01:23:43 +01001865phys_addr_t __init_memblock memblock_get_current_limit(void)
1866{
1867 return memblock.current_limit;
1868}
1869
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001870static void __init_memblock memblock_dump(struct memblock_type *type)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001871{
Miles Chen5d63f812017-02-22 15:46:42 -08001872 phys_addr_t base, end, size;
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001873 enum memblock_flags flags;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -08001874 int idx;
1875 struct memblock_region *rgn;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001876
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001877 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001878
Gioh Kim66e8b432017-11-15 17:33:42 -08001879 for_each_memblock_type(idx, type, rgn) {
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001880 char nid_buf[32] = "";
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001881
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001882 base = rgn->base;
1883 size = rgn->size;
Miles Chen5d63f812017-02-22 15:46:42 -08001884 end = base + size - 1;
Tang Chen66a20752014-01-21 15:49:20 -08001885 flags = rgn->flags;
Mike Rapoport3f08a302020-06-03 15:57:02 -07001886#ifdef CONFIG_NEED_MULTIPLE_NODES
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001887 if (memblock_get_region_node(rgn) != MAX_NUMNODES)
1888 snprintf(nid_buf, sizeof(nid_buf), " on node %d",
1889 memblock_get_region_node(rgn));
1890#endif
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001891 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001892 type->name, idx, &base, &end, &size, nid_buf, flags);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001893 }
1894}
1895
Mike Rapoport87c55872020-10-13 16:57:54 -07001896static void __init_memblock __memblock_dump_all(void)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001897{
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001898 pr_info("MEMBLOCK configuration:\n");
Miles Chen5d63f812017-02-22 15:46:42 -08001899 pr_info(" memory size = %pa reserved size = %pa\n",
1900 &memblock.memory.total_size,
1901 &memblock.reserved.total_size);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001902
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001903 memblock_dump(&memblock.memory);
1904 memblock_dump(&memblock.reserved);
Heiko Carstens409efd42017-02-24 14:55:56 -08001905#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
David Hildenbrand77649902020-07-01 16:18:29 +02001906 memblock_dump(&physmem);
Heiko Carstens409efd42017-02-24 14:55:56 -08001907#endif
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001908}
1909
Mike Rapoport87c55872020-10-13 16:57:54 -07001910void __init_memblock memblock_dump_all(void)
1911{
1912 if (memblock_debug)
1913 __memblock_dump_all();
1914}
1915
Tejun Heo1aadc052011-12-08 10:22:08 -08001916void __init memblock_allow_resize(void)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001917{
Benjamin Herrenschmidt142b45a2010-07-06 15:39:13 -07001918 memblock_can_resize = 1;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001919}
1920
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001921static int __init early_memblock(char *p)
1922{
1923 if (p && strstr(p, "debug"))
1924 memblock_debug = 1;
1925 return 0;
1926}
1927early_param("memblock", early_memblock);
1928
Mike Rapoportbda49a82018-10-30 15:09:40 -07001929static void __init __free_pages_memory(unsigned long start, unsigned long end)
1930{
1931 int order;
1932
1933 while (start < end) {
1934 order = min(MAX_ORDER - 1UL, __ffs(start));
1935
1936 while (start + (1UL << order) > end)
1937 order--;
1938
1939 memblock_free_pages(pfn_to_page(start), start, order);
1940
1941 start += (1UL << order);
1942 }
1943}
1944
1945static unsigned long __init __free_memory_core(phys_addr_t start,
1946 phys_addr_t end)
1947{
1948 unsigned long start_pfn = PFN_UP(start);
1949 unsigned long end_pfn = min_t(unsigned long,
1950 PFN_DOWN(end), max_low_pfn);
1951
1952 if (start_pfn >= end_pfn)
1953 return 0;
1954
1955 __free_pages_memory(start_pfn, end_pfn);
1956
1957 return end_pfn - start_pfn;
1958}
1959
1960static unsigned long __init free_low_memory_core_early(void)
1961{
1962 unsigned long count = 0;
1963 phys_addr_t start, end;
1964 u64 i;
1965
1966 memblock_clear_hotplug(0, -1);
1967
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -07001968 for_each_reserved_mem_range(i, &start, &end)
Mike Rapoportbda49a82018-10-30 15:09:40 -07001969 reserve_bootmem_region(start, end);
1970
1971 /*
1972 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
1973 * because in some case like Node0 doesn't have RAM installed
1974 * low ram will be on Node1
1975 */
1976 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
1977 NULL)
1978 count += __free_memory_core(start, end);
1979
1980 return count;
1981}
1982
1983static int reset_managed_pages_done __initdata;
1984
1985void reset_node_managed_pages(pg_data_t *pgdat)
1986{
1987 struct zone *z;
1988
1989 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
Arun KS9705bea2018-12-28 00:34:24 -08001990 atomic_long_set(&z->managed_pages, 0);
Mike Rapoportbda49a82018-10-30 15:09:40 -07001991}
1992
1993void __init reset_all_zones_managed_pages(void)
1994{
1995 struct pglist_data *pgdat;
1996
1997 if (reset_managed_pages_done)
1998 return;
1999
2000 for_each_online_pgdat(pgdat)
2001 reset_node_managed_pages(pgdat);
2002
2003 reset_managed_pages_done = 1;
2004}
2005
2006/**
2007 * memblock_free_all - release free pages to the buddy allocator
2008 *
2009 * Return: the number of pages actually released.
2010 */
2011unsigned long __init memblock_free_all(void)
2012{
2013 unsigned long pages;
2014
2015 reset_all_zones_managed_pages();
2016
2017 pages = free_low_memory_core_early();
Arun KSca79b0c2018-12-28 00:34:29 -08002018 totalram_pages_add(pages);
Mike Rapoportbda49a82018-10-30 15:09:40 -07002019
2020 return pages;
2021}
2022
Mike Rapoport350e88b2019-05-13 17:22:59 -07002023#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002024
2025static int memblock_debug_show(struct seq_file *m, void *private)
2026{
2027 struct memblock_type *type = m->private;
2028 struct memblock_region *reg;
2029 int i;
Miles Chen5d63f812017-02-22 15:46:42 -08002030 phys_addr_t end;
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002031
2032 for (i = 0; i < type->cnt; i++) {
2033 reg = &type->regions[i];
Miles Chen5d63f812017-02-22 15:46:42 -08002034 end = reg->base + reg->size - 1;
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002035
Miles Chen5d63f812017-02-22 15:46:42 -08002036 seq_printf(m, "%4d: ", i);
2037 seq_printf(m, "%pa..%pa\n", &reg->base, &end);
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002038 }
2039 return 0;
2040}
Andy Shevchenko5ad35092018-04-05 16:23:16 -07002041DEFINE_SHOW_ATTRIBUTE(memblock_debug);
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002042
2043static int __init memblock_init_debugfs(void)
2044{
2045 struct dentry *root = debugfs_create_dir("memblock", NULL);
Greg Kroah-Hartmand9f79792019-03-05 15:46:09 -08002046
Joe Perches0825a6f2018-06-14 15:27:58 -07002047 debugfs_create_file("memory", 0444, root,
2048 &memblock.memory, &memblock_debug_fops);
2049 debugfs_create_file("reserved", 0444, root,
2050 &memblock.reserved, &memblock_debug_fops);
Philipp Hachtmann70210ed2014-01-29 18:16:01 +01002051#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
David Hildenbrand77649902020-07-01 16:18:29 +02002052 debugfs_create_file("physmem", 0444, root, &physmem,
2053 &memblock_debug_fops);
Philipp Hachtmann70210ed2014-01-29 18:16:01 +01002054#endif
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002055
2056 return 0;
2057}
2058__initcall(memblock_init_debugfs);
2059
2060#endif /* CONFIG_DEBUG_FS */