blob: dc37b9bb8cb96bd39c5e34f9575aa7628e9de907 [file] [log] [blame]
Thomas Gleixner50acfb22019-05-29 07:18:00 -07001// SPDX-License-Identifier: GPL-2.0-only
Palmer Dabbelt76d2a042017-07-10 18:00:26 -07002/*
3 * Copyright (C) 2012 Regents of the University of California
Anup Patel671f9a32019-06-28 13:36:21 -07004 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
Nick Kossifidise53d2812021-04-19 03:55:38 +03005 * Copyright (C) 2020 FORTH-ICS/CARV
6 * Nick Kossifidis <mick@ics.forth.gr>
Palmer Dabbelt76d2a042017-07-10 18:00:26 -07007 */
8
9#include <linux/init.h>
10#include <linux/mm.h>
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070011#include <linux/memblock.h>
Mike Rapoport57c8a662018-10-30 15:09:49 -070012#include <linux/initrd.h>
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070013#include <linux/swap.h>
Christoph Hellwig5ec9c4f2018-01-16 09:37:50 +010014#include <linux/sizes.h>
Anup Patel0651c262019-02-21 11:25:49 +053015#include <linux/of_fdt.h>
Nick Kossifidis56409752021-04-19 03:55:39 +030016#include <linux/of_reserved_mem.h>
Albert Ou922b0372019-09-27 16:14:18 -070017#include <linux/libfdt.h>
Zong Lid27c3c92020-03-10 00:55:41 +080018#include <linux/set_memory.h>
Kefeng Wangda815582020-10-31 14:01:12 +080019#include <linux/dma-map-ops.h>
Nick Kossifidise53d2812021-04-19 03:55:38 +030020#include <linux/crash_dump.h>
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070021
Anup Patelf2c17aa2019-01-07 20:57:01 +053022#include <asm/fixmap.h>
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070023#include <asm/tlbflush.h>
24#include <asm/sections.h>
Palmer Dabbelt2d268252020-04-14 13:43:24 +090025#include <asm/soc.h>
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070026#include <asm/io.h>
Zong Lib422d282020-06-03 16:03:55 -070027#include <asm/ptdump.h>
Atish Patra4f0e8ee2020-11-18 16:38:29 -080028#include <asm/numa.h>
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070029
Paul Walmsleyffaee272019-10-17 15:00:17 -070030#include "../kernel/head.h"
31
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -040032unsigned long kernel_virt_addr = KERNEL_LINK_ADDR;
33EXPORT_SYMBOL(kernel_virt_addr);
Vitaly Wool44c92252021-04-13 02:35:14 -040034#ifdef CONFIG_XIP_KERNEL
35#define kernel_virt_addr (*((unsigned long *)XIP_FIXUP(&kernel_virt_addr)))
36#endif
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -040037
Anup Patel387181d2019-03-26 08:03:47 +000038unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
39 __page_aligned_bss;
40EXPORT_SYMBOL(empty_zero_page);
41
Anup Pateld90d45d2019-06-07 06:01:29 +000042extern char _start[];
Anup Patel8f3a2b42020-09-17 15:37:10 -070043#define DTB_EARLY_BASE_VA PGDIR_SIZE
Vitaly Wool44c92252021-04-13 02:35:14 -040044void *_dtb_early_va __initdata;
45uintptr_t _dtb_early_pa __initdata;
Anup Pateld90d45d2019-06-07 06:01:29 +000046
Atish Patrae8dcb612020-09-17 15:37:12 -070047struct pt_alloc_ops {
48 pte_t *(*get_pte_virt)(phys_addr_t pa);
49 phys_addr_t (*alloc_pte)(uintptr_t va);
50#ifndef __PAGETABLE_PMD_FOLDED
51 pmd_t *(*get_pmd_virt)(phys_addr_t pa);
52 phys_addr_t (*alloc_pmd)(uintptr_t va);
53#endif
54};
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070055
Kefeng Wangda815582020-10-31 14:01:12 +080056static phys_addr_t dma32_phys_limit __ro_after_init;
57
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070058static void __init zone_sizes_init(void)
59{
Christoph Hellwig5ec9c4f2018-01-16 09:37:50 +010060 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, };
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070061
Zong Lid5fad482018-06-25 16:49:37 +080062#ifdef CONFIG_ZONE_DMA32
Kefeng Wangda815582020-10-31 14:01:12 +080063 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit);
Zong Lid5fad482018-06-25 16:49:37 +080064#endif
Christoph Hellwig5ec9c4f2018-01-16 09:37:50 +010065 max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
66
Mike Rapoport9691a072020-06-03 15:57:10 -070067 free_area_init(max_zone_pfns);
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070068}
69
Jisheng Zhang19875012021-03-30 02:22:21 +080070static void __init setup_zero_page(void)
Palmer Dabbelt76d2a042017-07-10 18:00:26 -070071{
72 memset((void *)empty_zero_page, 0, PAGE_SIZE);
73}
74
Kefeng Wang8fa3cdf2020-05-14 19:53:35 +080075#if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM)
Yash Shah2cc6c4a2019-11-18 05:58:34 +000076static inline void print_mlk(char *name, unsigned long b, unsigned long t)
77{
78 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t,
79 (((t) - (b)) >> 10));
80}
81
82static inline void print_mlm(char *name, unsigned long b, unsigned long t)
83{
84 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t,
85 (((t) - (b)) >> 20));
86}
87
Jisheng Zhang19875012021-03-30 02:22:21 +080088static void __init print_vm_layout(void)
Yash Shah2cc6c4a2019-11-18 05:58:34 +000089{
90 pr_notice("Virtual kernel memory layout:\n");
91 print_mlk("fixmap", (unsigned long)FIXADDR_START,
92 (unsigned long)FIXADDR_TOP);
93 print_mlm("pci io", (unsigned long)PCI_IO_START,
94 (unsigned long)PCI_IO_END);
95 print_mlm("vmemmap", (unsigned long)VMEMMAP_START,
96 (unsigned long)VMEMMAP_END);
97 print_mlm("vmalloc", (unsigned long)VMALLOC_START,
98 (unsigned long)VMALLOC_END);
99 print_mlm("lowmem", (unsigned long)PAGE_OFFSET,
100 (unsigned long)high_memory);
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400101#ifdef CONFIG_64BIT
102 print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR,
103 (unsigned long)ADDRESS_SPACE_END);
104#endif
Yash Shah2cc6c4a2019-11-18 05:58:34 +0000105}
106#else
107static void print_vm_layout(void) { }
108#endif /* CONFIG_DEBUG_VM */
109
Palmer Dabbelt76d2a042017-07-10 18:00:26 -0700110void __init mem_init(void)
111{
112#ifdef CONFIG_FLATMEM
113 BUG_ON(!mem_map);
114#endif /* CONFIG_FLATMEM */
115
116 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn)));
Mike Rapoportc6ffc5c2018-10-30 15:09:30 -0700117 memblock_free_all();
Palmer Dabbelt76d2a042017-07-10 18:00:26 -0700118
Yash Shah2cc6c4a2019-11-18 05:58:34 +0000119 print_vm_layout();
Palmer Dabbelt76d2a042017-07-10 18:00:26 -0700120}
121
Anup Patel0651c262019-02-21 11:25:49 +0530122void __init setup_bootmem(void)
123{
Zong Liac51e002020-01-02 11:12:40 +0800124 phys_addr_t vmlinux_end = __pa_symbol(&_end);
125 phys_addr_t vmlinux_start = __pa_symbol(&_start);
Kefeng Wangdd2d0822021-02-09 09:01:51 +0800126 phys_addr_t dram_end = memblock_end_of_DRAM();
Atish Patraabb8e862021-01-11 15:45:02 -0800127 phys_addr_t max_mapped_addr = __pa(~(ulong)0);
Anup Patel0651c262019-02-21 11:25:49 +0530128
Vitaly Wool44c92252021-04-13 02:35:14 -0400129#ifdef CONFIG_XIP_KERNEL
130 vmlinux_start = __pa_symbol(&_sdata);
131#endif
132
Kefeng Wangdd2d0822021-02-09 09:01:51 +0800133 /* The maximal physical memory size is -PAGE_OFFSET. */
Atish Patrade043da2020-12-18 16:13:56 -0800134 memblock_enforce_memory_limit(-PAGE_OFFSET);
Anup Patel0651c262019-02-21 11:25:49 +0530135
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400136 /*
137 * Reserve from the start of the kernel to the end of the kernel
Geert Uytterhoeven8db6f932021-04-29 17:05:00 +0200138 */
139#if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX)
140 /*
141 * Make sure we align the reservation on PMD_SIZE since we will
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400142 * map the kernel in the linear mapping as read-only: we do not want
143 * any allocation to happen between _end and the next pmd aligned page.
144 */
Geert Uytterhoeven8db6f932021-04-29 17:05:00 +0200145 vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK;
146#endif
147 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start);
Anup Pateld90d45d2019-06-07 06:01:29 +0000148
Atish Patraabb8e862021-01-11 15:45:02 -0800149 /*
150 * memblock allocator is not aware of the fact that last 4K bytes of
151 * the addressable memory can not be mapped because of IS_ERR_VALUE
152 * macro. Make sure that last 4k bytes are not usable by memblock
153 * if end of dram is equal to maximum addressable memory.
154 */
155 if (max_mapped_addr == (dram_end - 1))
156 memblock_set_current_limit(max_mapped_addr - 4096);
157
Kefeng Wangf6e5aed2021-02-25 14:54:17 +0800158 min_low_pfn = PFN_UP(memblock_start_of_DRAM());
159 max_low_pfn = max_pfn = PFN_DOWN(dram_end);
160
Kefeng Wangda815582020-10-31 14:01:12 +0800161 dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn));
Guo Ren336e8eb2021-01-21 14:31:17 +0800162 set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET);
Anup Patel0651c262019-02-21 11:25:49 +0530163
Kefeng Wangaec33b52021-01-15 13:46:06 +0800164 reserve_initrd_mem();
Albert Ou922b0372019-09-27 16:14:18 -0700165 /*
Vitaly Woolf105aa92021-01-16 01:49:48 +0200166 * If DTB is built in, no need to reserve its memblock.
167 * Otherwise, do reserve it but avoid using
168 * early_init_fdt_reserve_self() since __pa() does
Albert Ou922b0372019-09-27 16:14:18 -0700169 * not work for DTB pointers that are fixmap addresses
170 */
Vitaly Woolf105aa92021-01-16 01:49:48 +0200171 if (!IS_ENABLED(CONFIG_BUILTIN_DTB))
172 memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va));
Albert Ou922b0372019-09-27 16:14:18 -0700173
Anup Patel0651c262019-02-21 11:25:49 +0530174 early_init_fdt_scan_reserved_mem();
Kefeng Wangda815582020-10-31 14:01:12 +0800175 dma_contiguous_reserve(dma32_phys_limit);
Anup Patel0651c262019-02-21 11:25:49 +0530176 memblock_allow_resize();
Anup Patel0651c262019-02-21 11:25:49 +0530177}
Anup Patel6f1e9e92019-02-13 16:38:36 +0530178
Vitaly Wool44c92252021-04-13 02:35:14 -0400179#ifdef CONFIG_XIP_KERNEL
Atish Patrae8dcb612020-09-17 15:37:12 -0700180
Vitaly Wool44c92252021-04-13 02:35:14 -0400181extern char _xiprom[], _exiprom[];
182extern char _sdata[], _edata[];
183
184#endif /* CONFIG_XIP_KERNEL */
185
Palmer Dabbelt76d2a042017-07-10 18:00:26 -0700186#ifdef CONFIG_MMU
Vitaly Wool44c92252021-04-13 02:35:14 -0400187static struct pt_alloc_ops _pt_ops __ro_after_init;
188
189#ifdef CONFIG_XIP_KERNEL
190#define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&_pt_ops))
191#else
192#define pt_ops _pt_ops
193#endif
Christoph Hellwig6bd33e12019-10-28 13:10:41 +0100194
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400195/* Offset between linear mapping virtual address and kernel load address */
Jisheng Zhangde31ea42021-03-30 02:22:51 +0800196unsigned long va_pa_offset __ro_after_init;
Anup Patel387181d2019-03-26 08:03:47 +0000197EXPORT_SYMBOL(va_pa_offset);
Vitaly Wool44c92252021-04-13 02:35:14 -0400198#ifdef CONFIG_XIP_KERNEL
199#define va_pa_offset (*((unsigned long *)XIP_FIXUP(&va_pa_offset)))
200#endif
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400201/* Offset between kernel mapping virtual address and kernel load address */
Vitaly Wool44c92252021-04-13 02:35:14 -0400202#ifdef CONFIG_64BIT
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400203unsigned long va_kernel_pa_offset;
204EXPORT_SYMBOL(va_kernel_pa_offset);
205#endif
Vitaly Wool44c92252021-04-13 02:35:14 -0400206#ifdef CONFIG_XIP_KERNEL
207#define va_kernel_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_pa_offset)))
208#endif
209unsigned long va_kernel_xip_pa_offset;
210EXPORT_SYMBOL(va_kernel_xip_pa_offset);
211#ifdef CONFIG_XIP_KERNEL
212#define va_kernel_xip_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_xip_pa_offset)))
213#endif
Jisheng Zhangde31ea42021-03-30 02:22:51 +0800214unsigned long pfn_base __ro_after_init;
Anup Patel387181d2019-03-26 08:03:47 +0000215EXPORT_SYMBOL(pfn_base);
216
Anup Patel6f1e9e92019-02-13 16:38:36 +0530217pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
Anup Patel671f9a32019-06-28 13:36:21 -0700218pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
Anup Patelf2c17aa2019-01-07 20:57:01 +0530219pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;
Anup Patel671f9a32019-06-28 13:36:21 -0700220
Anup Patel671f9a32019-06-28 13:36:21 -0700221pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
Anup Patelf2c17aa2019-01-07 20:57:01 +0530222
Vitaly Wool44c92252021-04-13 02:35:14 -0400223#ifdef CONFIG_XIP_KERNEL
224#define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir))
225#define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte))
226#define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir))
227#endif /* CONFIG_XIP_KERNEL */
228
Anup Patelf2c17aa2019-01-07 20:57:01 +0530229void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
230{
231 unsigned long addr = __fix_to_virt(idx);
232 pte_t *ptep;
233
234 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
235
236 ptep = &fixmap_pte[pte_index(addr)];
237
Greentime Hu21190b72020-08-04 11:02:05 +0800238 if (pgprot_val(prot))
Anup Patelf2c17aa2019-01-07 20:57:01 +0530239 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot));
Greentime Hu21190b72020-08-04 11:02:05 +0800240 else
Anup Patelf2c17aa2019-01-07 20:57:01 +0530241 pte_clear(&init_mm, addr, ptep);
Greentime Hu21190b72020-08-04 11:02:05 +0800242 local_flush_tlb_page(addr);
Anup Patelf2c17aa2019-01-07 20:57:01 +0530243}
244
Atish Patrae8dcb612020-09-17 15:37:12 -0700245static inline pte_t *__init get_pte_virt_early(phys_addr_t pa)
Anup Patel671f9a32019-06-28 13:36:21 -0700246{
Atish Patrae8dcb612020-09-17 15:37:12 -0700247 return (pte_t *)((uintptr_t)pa);
Anup Patel671f9a32019-06-28 13:36:21 -0700248}
249
Atish Patrae8dcb612020-09-17 15:37:12 -0700250static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa)
251{
252 clear_fixmap(FIX_PTE);
253 return (pte_t *)set_fixmap_offset(FIX_PTE, pa);
254}
255
256static inline pte_t *get_pte_virt_late(phys_addr_t pa)
257{
258 return (pte_t *) __va(pa);
259}
260
261static inline phys_addr_t __init alloc_pte_early(uintptr_t va)
Anup Patel671f9a32019-06-28 13:36:21 -0700262{
263 /*
264 * We only create PMD or PGD early mappings so we
265 * should never reach here with MMU disabled.
266 */
Atish Patrae8dcb612020-09-17 15:37:12 -0700267 BUG();
268}
Anup Patel671f9a32019-06-28 13:36:21 -0700269
Atish Patrae8dcb612020-09-17 15:37:12 -0700270static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va)
271{
Anup Patel671f9a32019-06-28 13:36:21 -0700272 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
273}
274
Atish Patrae8dcb612020-09-17 15:37:12 -0700275static phys_addr_t alloc_pte_late(uintptr_t va)
276{
277 unsigned long vaddr;
278
279 vaddr = __get_free_page(GFP_KERNEL);
zhouchuangaoe75e6bf2021-03-30 06:56:26 -0700280 BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr)));
281
Atish Patrae8dcb612020-09-17 15:37:12 -0700282 return __pa(vaddr);
283}
284
Anup Patel671f9a32019-06-28 13:36:21 -0700285static void __init create_pte_mapping(pte_t *ptep,
286 uintptr_t va, phys_addr_t pa,
287 phys_addr_t sz, pgprot_t prot)
288{
Mike Rapoport974b9b22020-06-08 21:33:10 -0700289 uintptr_t pte_idx = pte_index(va);
Anup Patel671f9a32019-06-28 13:36:21 -0700290
291 BUG_ON(sz != PAGE_SIZE);
292
Mike Rapoport974b9b22020-06-08 21:33:10 -0700293 if (pte_none(ptep[pte_idx]))
294 ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot);
Anup Patel671f9a32019-06-28 13:36:21 -0700295}
296
297#ifndef __PAGETABLE_PMD_FOLDED
298
299pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss;
300pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss;
Alexandre Ghiti0f02de42021-02-21 09:22:33 -0500301pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
Anup Patel1074dd42020-11-04 12:07:13 +0530302pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
Anup Patel671f9a32019-06-28 13:36:21 -0700303
Vitaly Wool44c92252021-04-13 02:35:14 -0400304#ifdef CONFIG_XIP_KERNEL
305#define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd))
306#define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd))
307#define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd))
308#endif /* CONFIG_XIP_KERNEL */
309
Atish Patrae8dcb612020-09-17 15:37:12 -0700310static pmd_t *__init get_pmd_virt_early(phys_addr_t pa)
Anup Patel671f9a32019-06-28 13:36:21 -0700311{
Atish Patrae8dcb612020-09-17 15:37:12 -0700312 /* Before MMU is enabled */
313 return (pmd_t *)((uintptr_t)pa);
Anup Patel671f9a32019-06-28 13:36:21 -0700314}
315
Atish Patrae8dcb612020-09-17 15:37:12 -0700316static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa)
317{
318 clear_fixmap(FIX_PMD);
319 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa);
320}
321
322static pmd_t *get_pmd_virt_late(phys_addr_t pa)
323{
324 return (pmd_t *) __va(pa);
325}
326
327static phys_addr_t __init alloc_pmd_early(uintptr_t va)
Anup Patel671f9a32019-06-28 13:36:21 -0700328{
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400329 BUG_ON((va - kernel_virt_addr) >> PGDIR_SHIFT);
Anup Patel671f9a32019-06-28 13:36:21 -0700330
Alexandre Ghiti0f02de42021-02-21 09:22:33 -0500331 return (uintptr_t)early_pmd;
Anup Patel671f9a32019-06-28 13:36:21 -0700332}
333
Atish Patrae8dcb612020-09-17 15:37:12 -0700334static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va)
335{
336 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
337}
338
339static phys_addr_t alloc_pmd_late(uintptr_t va)
340{
341 unsigned long vaddr;
342
343 vaddr = __get_free_page(GFP_KERNEL);
344 BUG_ON(!vaddr);
345 return __pa(vaddr);
346}
347
Anup Patel671f9a32019-06-28 13:36:21 -0700348static void __init create_pmd_mapping(pmd_t *pmdp,
349 uintptr_t va, phys_addr_t pa,
350 phys_addr_t sz, pgprot_t prot)
351{
352 pte_t *ptep;
353 phys_addr_t pte_phys;
Mike Rapoport974b9b22020-06-08 21:33:10 -0700354 uintptr_t pmd_idx = pmd_index(va);
Anup Patel671f9a32019-06-28 13:36:21 -0700355
356 if (sz == PMD_SIZE) {
Mike Rapoport974b9b22020-06-08 21:33:10 -0700357 if (pmd_none(pmdp[pmd_idx]))
358 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot);
Anup Patel671f9a32019-06-28 13:36:21 -0700359 return;
360 }
361
Mike Rapoport974b9b22020-06-08 21:33:10 -0700362 if (pmd_none(pmdp[pmd_idx])) {
Atish Patrae8dcb612020-09-17 15:37:12 -0700363 pte_phys = pt_ops.alloc_pte(va);
Mike Rapoport974b9b22020-06-08 21:33:10 -0700364 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE);
Atish Patrae8dcb612020-09-17 15:37:12 -0700365 ptep = pt_ops.get_pte_virt(pte_phys);
Anup Patel671f9a32019-06-28 13:36:21 -0700366 memset(ptep, 0, PAGE_SIZE);
367 } else {
Mike Rapoport974b9b22020-06-08 21:33:10 -0700368 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx]));
Atish Patrae8dcb612020-09-17 15:37:12 -0700369 ptep = pt_ops.get_pte_virt(pte_phys);
Anup Patel671f9a32019-06-28 13:36:21 -0700370 }
371
372 create_pte_mapping(ptep, va, pa, sz, prot);
373}
374
375#define pgd_next_t pmd_t
Atish Patrae8dcb612020-09-17 15:37:12 -0700376#define alloc_pgd_next(__va) pt_ops.alloc_pmd(__va)
377#define get_pgd_next_virt(__pa) pt_ops.get_pmd_virt(__pa)
Anup Patel671f9a32019-06-28 13:36:21 -0700378#define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
379 create_pmd_mapping(__nextp, __va, __pa, __sz, __prot)
Anup Patel671f9a32019-06-28 13:36:21 -0700380#define fixmap_pgd_next fixmap_pmd
381#else
382#define pgd_next_t pte_t
Atish Patrae8dcb612020-09-17 15:37:12 -0700383#define alloc_pgd_next(__va) pt_ops.alloc_pte(__va)
384#define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa)
Anup Patel671f9a32019-06-28 13:36:21 -0700385#define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
386 create_pte_mapping(__nextp, __va, __pa, __sz, __prot)
Anup Patel671f9a32019-06-28 13:36:21 -0700387#define fixmap_pgd_next fixmap_pte
388#endif
389
Atish Patrab91540d2020-09-17 15:37:15 -0700390void __init create_pgd_mapping(pgd_t *pgdp,
Anup Patel671f9a32019-06-28 13:36:21 -0700391 uintptr_t va, phys_addr_t pa,
392 phys_addr_t sz, pgprot_t prot)
393{
394 pgd_next_t *nextp;
395 phys_addr_t next_phys;
Mike Rapoport974b9b22020-06-08 21:33:10 -0700396 uintptr_t pgd_idx = pgd_index(va);
Anup Patel671f9a32019-06-28 13:36:21 -0700397
398 if (sz == PGDIR_SIZE) {
Mike Rapoport974b9b22020-06-08 21:33:10 -0700399 if (pgd_val(pgdp[pgd_idx]) == 0)
400 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot);
Anup Patel671f9a32019-06-28 13:36:21 -0700401 return;
402 }
403
Mike Rapoport974b9b22020-06-08 21:33:10 -0700404 if (pgd_val(pgdp[pgd_idx]) == 0) {
Anup Patel671f9a32019-06-28 13:36:21 -0700405 next_phys = alloc_pgd_next(va);
Mike Rapoport974b9b22020-06-08 21:33:10 -0700406 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE);
Anup Patel671f9a32019-06-28 13:36:21 -0700407 nextp = get_pgd_next_virt(next_phys);
408 memset(nextp, 0, PAGE_SIZE);
409 } else {
Mike Rapoport974b9b22020-06-08 21:33:10 -0700410 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx]));
Anup Patel671f9a32019-06-28 13:36:21 -0700411 nextp = get_pgd_next_virt(next_phys);
412 }
413
414 create_pgd_next_mapping(nextp, va, pa, sz, prot);
415}
416
417static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)
418{
Zong Li0fdc6362019-11-08 01:00:40 -0800419 /* Upgrade to PMD_SIZE mappings whenever possible */
420 if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1)))
421 return PAGE_SIZE;
Anup Patel671f9a32019-06-28 13:36:21 -0700422
Zong Li0fdc6362019-11-08 01:00:40 -0800423 return PMD_SIZE;
Anup Patel671f9a32019-06-28 13:36:21 -0700424}
425
Vitaly Wool44c92252021-04-13 02:35:14 -0400426#ifdef CONFIG_XIP_KERNEL
427/* called from head.S with MMU off */
428asmlinkage void __init __copy_data(void)
429{
430 void *from = (void *)(&_sdata);
431 void *end = (void *)(&_end);
432 void *to = (void *)CONFIG_PHYS_RAM_BASE;
433 size_t sz = (size_t)(end - from + 1);
434
435 memcpy(to, from, sz);
436}
437#endif
438
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200439#ifdef CONFIG_STRICT_KERNEL_RWX
440static __init pgprot_t pgprot_from_va(uintptr_t va)
441{
442 if (is_va_kernel_text(va))
443 return PAGE_KERNEL_READ_EXEC;
444
445 /*
446 * In 64-bit kernel, the kernel mapping is outside the linear mapping so
447 * we must protect its linear mapping alias from being executed and
448 * written.
449 * And rodata section is marked readonly in mark_rodata_ro.
450 */
451 if (IS_ENABLED(CONFIG_64BIT) && is_va_kernel_lm_alias_text(va))
452 return PAGE_KERNEL_READ;
453
454 return PAGE_KERNEL;
455}
456
457void mark_rodata_ro(void)
458{
459 set_kernel_memory(__start_rodata, _data, set_memory_ro);
460 if (IS_ENABLED(CONFIG_64BIT))
461 set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data),
462 set_memory_ro);
463
464 debug_checkwx();
465}
466#else
467static __init pgprot_t pgprot_from_va(uintptr_t va)
468{
469 if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va))
470 return PAGE_KERNEL;
471
472 return PAGE_KERNEL_EXEC;
473}
474#endif /* CONFIG_STRICT_KERNEL_RWX */
475
Anup Patel387181d2019-03-26 08:03:47 +0000476/*
477 * setup_vm() is called from head.S with MMU-off.
478 *
479 * Following requirements should be honoured for setup_vm() to work
480 * correctly:
481 * 1) It should use PC-relative addressing for accessing kernel symbols.
482 * To achieve this we always use GCC cmodel=medany.
483 * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
484 * so disable compiler instrumentation when FTRACE is enabled.
485 *
486 * Currently, the above requirements are honoured by using custom CFLAGS
487 * for init.o in mm/Makefile.
488 */
489
490#ifndef __riscv_cmodel_medany
Paul Walmsley6a527b62019-10-17 14:45:58 -0700491#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."
Anup Patel387181d2019-03-26 08:03:47 +0000492#endif
493
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200494static uintptr_t load_pa __initdata;
495uintptr_t load_sz;
Vitaly Wool44c92252021-04-13 02:35:14 -0400496#ifdef CONFIG_XIP_KERNEL
497#define load_pa (*((uintptr_t *)XIP_FIXUP(&load_pa)))
498#define load_sz (*((uintptr_t *)XIP_FIXUP(&load_sz)))
499#endif
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400500
Vitaly Wool44c92252021-04-13 02:35:14 -0400501#ifdef CONFIG_XIP_KERNEL
502uintptr_t xiprom, xiprom_sz;
503#define xiprom_sz (*((uintptr_t *)XIP_FIXUP(&xiprom_sz)))
504#define xiprom (*((uintptr_t *)XIP_FIXUP(&xiprom)))
505
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200506static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size,
507 __always_unused bool early)
Vitaly Wool44c92252021-04-13 02:35:14 -0400508{
509 uintptr_t va, end_va;
510
511 /* Map the flash resident part */
512 end_va = kernel_virt_addr + xiprom_sz;
513 for (va = kernel_virt_addr; va < end_va; va += map_size)
514 create_pgd_mapping(pgdir, va,
515 xiprom + (va - kernel_virt_addr),
516 map_size, PAGE_KERNEL_EXEC);
517
518 /* Map the data in RAM */
519 end_va = kernel_virt_addr + XIP_OFFSET + load_sz;
520 for (va = kernel_virt_addr + XIP_OFFSET; va < end_va; va += map_size)
521 create_pgd_mapping(pgdir, va,
522 load_pa + (va - (kernel_virt_addr + XIP_OFFSET)),
523 map_size, PAGE_KERNEL);
524}
525#else
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200526static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size,
527 bool early)
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400528{
529 uintptr_t va, end_va;
530
531 end_va = kernel_virt_addr + load_sz;
532 for (va = kernel_virt_addr; va < end_va; va += map_size)
533 create_pgd_mapping(pgdir, va,
534 load_pa + (va - kernel_virt_addr),
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200535 map_size,
536 early ?
537 PAGE_KERNEL_EXEC : pgprot_from_va(va));
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400538}
Vitaly Wool44c92252021-04-13 02:35:14 -0400539#endif
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400540
Anup Patel671f9a32019-06-28 13:36:21 -0700541asmlinkage void __init setup_vm(uintptr_t dtb_pa)
Anup Patel6f1e9e92019-02-13 16:38:36 +0530542{
Vitaly Wool44c92252021-04-13 02:35:14 -0400543 uintptr_t __maybe_unused pa;
Alexandre Ghiti0f02de42021-02-21 09:22:33 -0500544 uintptr_t map_size;
Atish Patra6262f662020-09-17 15:37:11 -0700545#ifndef __PAGETABLE_PMD_FOLDED
546 pmd_t fix_bmap_spmd, fix_bmap_epmd;
547#endif
Anup Patel6f1e9e92019-02-13 16:38:36 +0530548
Vitaly Wool44c92252021-04-13 02:35:14 -0400549#ifdef CONFIG_XIP_KERNEL
550 xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR;
551 xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom);
552
553 load_pa = (uintptr_t)CONFIG_PHYS_RAM_BASE;
554 load_sz = (uintptr_t)(&_end) - (uintptr_t)(&_sdata);
555
556 va_kernel_xip_pa_offset = kernel_virt_addr - xiprom;
557#else
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400558 load_pa = (uintptr_t)(&_start);
559 load_sz = (uintptr_t)(&_end) - load_pa;
Vitaly Wool44c92252021-04-13 02:35:14 -0400560#endif
Anup Patel6f1e9e92019-02-13 16:38:36 +0530561
Anup Patel671f9a32019-06-28 13:36:21 -0700562 va_pa_offset = PAGE_OFFSET - load_pa;
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400563#ifdef CONFIG_64BIT
564 va_kernel_pa_offset = kernel_virt_addr - load_pa;
565#endif
566
Anup Patel671f9a32019-06-28 13:36:21 -0700567 pfn_base = PFN_DOWN(load_pa);
568
569 /*
570 * Enforce boot alignment requirements of RV32 and
571 * RV64 by only allowing PMD or PGD mappings.
572 */
Alexandre Ghiti0f02de42021-02-21 09:22:33 -0500573 map_size = PMD_SIZE;
Anup Patel6f1e9e92019-02-13 16:38:36 +0530574
575 /* Sanity check alignment and size */
576 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0);
Anup Patel671f9a32019-06-28 13:36:21 -0700577 BUG_ON((load_pa % map_size) != 0);
Anup Patel671f9a32019-06-28 13:36:21 -0700578
Atish Patrae8dcb612020-09-17 15:37:12 -0700579 pt_ops.alloc_pte = alloc_pte_early;
580 pt_ops.get_pte_virt = get_pte_virt_early;
581#ifndef __PAGETABLE_PMD_FOLDED
582 pt_ops.alloc_pmd = alloc_pmd_early;
583 pt_ops.get_pmd_virt = get_pmd_virt_early;
584#endif
Anup Patel671f9a32019-06-28 13:36:21 -0700585 /* Setup early PGD for fixmap */
586 create_pgd_mapping(early_pg_dir, FIXADDR_START,
587 (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE);
Anup Patel6f1e9e92019-02-13 16:38:36 +0530588
589#ifndef __PAGETABLE_PMD_FOLDED
Anup Patel671f9a32019-06-28 13:36:21 -0700590 /* Setup fixmap PMD */
591 create_pmd_mapping(fixmap_pmd, FIXADDR_START,
592 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE);
593 /* Setup trampoline PGD and PMD */
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400594 create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr,
Anup Patel671f9a32019-06-28 13:36:21 -0700595 (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE);
Vitaly Wool44c92252021-04-13 02:35:14 -0400596#ifdef CONFIG_XIP_KERNEL
597 create_pmd_mapping(trampoline_pmd, kernel_virt_addr,
598 xiprom, PMD_SIZE, PAGE_KERNEL_EXEC);
599#else
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400600 create_pmd_mapping(trampoline_pmd, kernel_virt_addr,
Anup Patel671f9a32019-06-28 13:36:21 -0700601 load_pa, PMD_SIZE, PAGE_KERNEL_EXEC);
Vitaly Wool44c92252021-04-13 02:35:14 -0400602#endif
Anup Patel6f1e9e92019-02-13 16:38:36 +0530603#else
Anup Patel671f9a32019-06-28 13:36:21 -0700604 /* Setup trampoline PGD */
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400605 create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr,
Anup Patel671f9a32019-06-28 13:36:21 -0700606 load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC);
607#endif
Anup Patel6f1e9e92019-02-13 16:38:36 +0530608
Anup Patel671f9a32019-06-28 13:36:21 -0700609 /*
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400610 * Setup early PGD covering entire kernel which will allow
Anup Patel671f9a32019-06-28 13:36:21 -0700611 * us to reach paging_init(). We map all memory banks later
612 * in setup_vm_final() below.
613 */
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200614 create_kernel_page_table(early_pg_dir, map_size, true);
Anup Patelf2c17aa2019-01-07 20:57:01 +0530615
Anup Patel1074dd42020-11-04 12:07:13 +0530616#ifndef __PAGETABLE_PMD_FOLDED
617 /* Setup early PMD for DTB */
618 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA,
619 (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE);
Vitaly Woolf105aa92021-01-16 01:49:48 +0200620#ifndef CONFIG_BUILTIN_DTB
Anup Patel1074dd42020-11-04 12:07:13 +0530621 /* Create two consecutive PMD mappings for FDT early scan */
622 pa = dtb_pa & ~(PMD_SIZE - 1);
623 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA,
624 pa, PMD_SIZE, PAGE_KERNEL);
625 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE,
626 pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL);
627 dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1));
Vitaly Woolf105aa92021-01-16 01:49:48 +0200628#else /* CONFIG_BUILTIN_DTB */
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400629#ifdef CONFIG_64BIT
630 /*
631 * __va can't be used since it would return a linear mapping address
632 * whereas dtb_early_va will be used before setup_vm_final installs
633 * the linear mapping.
634 */
Vitaly Wool44c92252021-04-13 02:35:14 -0400635 dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa));
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400636#else
Vitaly Woolf105aa92021-01-16 01:49:48 +0200637 dtb_early_va = __va(dtb_pa);
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400638#endif /* CONFIG_64BIT */
Vitaly Woolf105aa92021-01-16 01:49:48 +0200639#endif /* CONFIG_BUILTIN_DTB */
Anup Patel1074dd42020-11-04 12:07:13 +0530640#else
Vitaly Woolf105aa92021-01-16 01:49:48 +0200641#ifndef CONFIG_BUILTIN_DTB
Anup Patel8f3a2b42020-09-17 15:37:10 -0700642 /* Create two consecutive PGD mappings for FDT early scan */
643 pa = dtb_pa & ~(PGDIR_SIZE - 1);
644 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA,
645 pa, PGDIR_SIZE, PAGE_KERNEL);
646 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE,
647 pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL);
648 dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1));
Vitaly Woolf105aa92021-01-16 01:49:48 +0200649#else /* CONFIG_BUILTIN_DTB */
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400650#ifdef CONFIG_64BIT
Vitaly Wool44c92252021-04-13 02:35:14 -0400651 dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa));
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400652#else
Vitaly Woolf105aa92021-01-16 01:49:48 +0200653 dtb_early_va = __va(dtb_pa);
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400654#endif /* CONFIG_64BIT */
Vitaly Woolf105aa92021-01-16 01:49:48 +0200655#endif /* CONFIG_BUILTIN_DTB */
Anup Patel1074dd42020-11-04 12:07:13 +0530656#endif
Albert Ou922b0372019-09-27 16:14:18 -0700657 dtb_early_pa = dtb_pa;
Atish Patra6262f662020-09-17 15:37:11 -0700658
659 /*
660 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap
661 * range can not span multiple pmds.
662 */
663 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
664 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
665
666#ifndef __PAGETABLE_PMD_FOLDED
667 /*
668 * Early ioremap fixmap is already created as it lies within first 2MB
669 * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END
670 * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn
671 * the user if not.
672 */
673 fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))];
674 fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))];
675 if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) {
676 WARN_ON(1);
677 pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n",
678 pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd));
679 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
680 fix_to_virt(FIX_BTMAP_BEGIN));
681 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n",
682 fix_to_virt(FIX_BTMAP_END));
683
684 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
685 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN);
686 }
687#endif
Anup Patel671f9a32019-06-28 13:36:21 -0700688}
689
690static void __init setup_vm_final(void)
691{
692 uintptr_t va, map_size;
693 phys_addr_t pa, start, end;
Mike Rapoportb10d6bc2020-10-13 16:58:08 -0700694 u64 i;
Anup Patel671f9a32019-06-28 13:36:21 -0700695
Atish Patrae8dcb612020-09-17 15:37:12 -0700696 /**
697 * MMU is enabled at this point. But page table setup is not complete yet.
698 * fixmap page table alloc functions should be used at this point
699 */
700 pt_ops.alloc_pte = alloc_pte_fixmap;
701 pt_ops.get_pte_virt = get_pte_virt_fixmap;
702#ifndef __PAGETABLE_PMD_FOLDED
703 pt_ops.alloc_pmd = alloc_pmd_fixmap;
704 pt_ops.get_pmd_virt = get_pmd_virt_fixmap;
705#endif
Anup Patel671f9a32019-06-28 13:36:21 -0700706 /* Setup swapper PGD for fixmap */
707 create_pgd_mapping(swapper_pg_dir, FIXADDR_START,
Zong Liac51e002020-01-02 11:12:40 +0800708 __pa_symbol(fixmap_pgd_next),
Anup Patel671f9a32019-06-28 13:36:21 -0700709 PGDIR_SIZE, PAGE_TABLE);
710
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400711 /* Map all memory banks in the linear mapping */
Mike Rapoportb10d6bc2020-10-13 16:58:08 -0700712 for_each_mem_range(i, &start, &end) {
Anup Patel671f9a32019-06-28 13:36:21 -0700713 if (start >= end)
714 break;
Anup Patel671f9a32019-06-28 13:36:21 -0700715 if (start <= __pa(PAGE_OFFSET) &&
716 __pa(PAGE_OFFSET) < end)
717 start = __pa(PAGE_OFFSET);
718
719 map_size = best_map_size(start, end - start);
720 for (pa = start; pa < end; pa += map_size) {
721 va = (uintptr_t)__va(pa);
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400722
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200723 create_pgd_mapping(swapper_pg_dir, va, pa, map_size,
724 pgprot_from_va(va));
Anup Patel671f9a32019-06-28 13:36:21 -0700725 }
Anup Patel6f1e9e92019-02-13 16:38:36 +0530726 }
Anup Patelf2c17aa2019-01-07 20:57:01 +0530727
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400728#ifdef CONFIG_64BIT
729 /* Map the kernel */
Alexandre Ghitie5c35fa02021-06-24 14:00:41 +0200730 create_kernel_page_table(swapper_pg_dir, PMD_SIZE, false);
Alexandre Ghiti2bfc6cd2021-04-11 12:41:44 -0400731#endif
732
Anup Patel671f9a32019-06-28 13:36:21 -0700733 /* Clear fixmap PTE and PMD mappings */
734 clear_fixmap(FIX_PTE);
735 clear_fixmap(FIX_PMD);
736
737 /* Move to swapper page table */
Zong Liac51e002020-01-02 11:12:40 +0800738 csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE);
Anup Patel671f9a32019-06-28 13:36:21 -0700739 local_flush_tlb_all();
Atish Patrae8dcb612020-09-17 15:37:12 -0700740
741 /* generic page allocation functions must be used to setup page table */
742 pt_ops.alloc_pte = alloc_pte_late;
743 pt_ops.get_pte_virt = get_pte_virt_late;
744#ifndef __PAGETABLE_PMD_FOLDED
745 pt_ops.alloc_pmd = alloc_pmd_late;
746 pt_ops.get_pmd_virt = get_pmd_virt_late;
747#endif
Anup Patel671f9a32019-06-28 13:36:21 -0700748}
Christoph Hellwig6bd33e12019-10-28 13:10:41 +0100749#else
750asmlinkage void __init setup_vm(uintptr_t dtb_pa)
751{
752 dtb_early_va = (void *)dtb_pa;
Atish Patraa78c6f52020-10-01 12:04:56 -0700753 dtb_early_pa = dtb_pa;
Christoph Hellwig6bd33e12019-10-28 13:10:41 +0100754}
755
756static inline void setup_vm_final(void)
757{
758}
759#endif /* CONFIG_MMU */
Anup Patel671f9a32019-06-28 13:36:21 -0700760
Nick Kossifidise53d2812021-04-19 03:55:38 +0300761#ifdef CONFIG_KEXEC_CORE
762/*
763 * reserve_crashkernel() - reserves memory for crash kernel
764 *
765 * This function reserves memory area given in "crashkernel=" kernel command
766 * line parameter. The memory reserved is used by dump capture kernel when
767 * primary kernel is crashing.
768 */
769static void __init reserve_crashkernel(void)
770{
771 unsigned long long crash_base = 0;
772 unsigned long long crash_size = 0;
773 unsigned long search_start = memblock_start_of_DRAM();
774 unsigned long search_end = memblock_end_of_DRAM();
775
776 int ret = 0;
777
Nick Kossifidis56409752021-04-19 03:55:39 +0300778 /*
779 * Don't reserve a region for a crash kernel on a crash kernel
780 * since it doesn't make much sense and we have limited memory
781 * resources.
782 */
783#ifdef CONFIG_CRASH_DUMP
784 if (is_kdump_kernel()) {
785 pr_info("crashkernel: ignoring reservation request\n");
786 return;
787 }
788#endif
789
Nick Kossifidise53d2812021-04-19 03:55:38 +0300790 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
791 &crash_size, &crash_base);
792 if (ret || !crash_size)
793 return;
794
795 crash_size = PAGE_ALIGN(crash_size);
796
797 if (crash_base == 0) {
798 /*
799 * Current riscv boot protocol requires 2MB alignment for
800 * RV64 and 4MB alignment for RV32 (hugepage size)
801 */
802 crash_base = memblock_find_in_range(search_start, search_end,
803 crash_size, PMD_SIZE);
804
805 if (crash_base == 0) {
806 pr_warn("crashkernel: couldn't allocate %lldKB\n",
807 crash_size >> 10);
808 return;
809 }
810 } else {
811 /* User specifies base address explicitly. */
812 if (!memblock_is_region_memory(crash_base, crash_size)) {
813 pr_warn("crashkernel: requested region is not memory\n");
814 return;
815 }
816
817 if (memblock_is_region_reserved(crash_base, crash_size)) {
818 pr_warn("crashkernel: requested region is reserved\n");
819 return;
820 }
821
822
823 if (!IS_ALIGNED(crash_base, PMD_SIZE)) {
824 pr_warn("crashkernel: requested region is misaligned\n");
825 return;
826 }
827 }
828 memblock_reserve(crash_base, crash_size);
829
830 pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n",
831 crash_base, crash_base + crash_size, crash_size >> 20);
832
833 crashk_res.start = crash_base;
834 crashk_res.end = crash_base + crash_size - 1;
835}
836#endif /* CONFIG_KEXEC_CORE */
837
Nick Kossifidis56409752021-04-19 03:55:39 +0300838#ifdef CONFIG_CRASH_DUMP
839/*
840 * We keep track of the ELF core header of the crashed
841 * kernel with a reserved-memory region with compatible
842 * string "linux,elfcorehdr". Here we register a callback
843 * to populate elfcorehdr_addr/size when this region is
844 * present. Note that this region will be marked as
845 * reserved once we call early_init_fdt_scan_reserved_mem()
846 * later on.
847 */
848static int elfcore_hdr_setup(struct reserved_mem *rmem)
849{
850 elfcorehdr_addr = rmem->base;
851 elfcorehdr_size = rmem->size;
852 return 0;
853}
854
855RESERVEDMEM_OF_DECLARE(elfcorehdr, "linux,elfcorehdr", elfcore_hdr_setup);
856#endif
857
Anup Patel671f9a32019-06-28 13:36:21 -0700858void __init paging_init(void)
859{
860 setup_vm_final();
861 setup_zero_page();
Atish Patracbd34f42020-11-18 16:38:27 -0800862}
863
864void __init misc_mem_init(void)
865{
Kefeng Wangf6e5aed2021-02-25 14:54:17 +0800866 early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT);
Atish Patra4f0e8ee2020-11-18 16:38:29 -0800867 arch_numa_init();
Atish Patracbd34f42020-11-18 16:38:27 -0800868 sparse_init();
Anup Patel671f9a32019-06-28 13:36:21 -0700869 zone_sizes_init();
Nick Kossifidise53d2812021-04-19 03:55:38 +0300870#ifdef CONFIG_KEXEC_CORE
871 reserve_crashkernel();
872#endif
Atish Patra4f0e8ee2020-11-18 16:38:29 -0800873 memblock_dump_all();
Anup Patel6f1e9e92019-02-13 16:38:36 +0530874}
Logan Gunthorped95f1a52019-08-28 15:40:54 -0600875
Kefeng Wang9fe57d82019-10-23 11:23:02 +0800876#ifdef CONFIG_SPARSEMEM_VMEMMAP
Logan Gunthorped95f1a52019-08-28 15:40:54 -0600877int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
878 struct vmem_altmap *altmap)
879{
Anshuman Khandual1d9cfee2020-08-06 23:23:19 -0700880 return vmemmap_populate_basepages(start, end, node, NULL);
Logan Gunthorped95f1a52019-08-28 15:40:54 -0600881}
882#endif