Thomas Gleixner | 50acfb2 | 2019-05-29 07:18:00 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 2012 Regents of the University of California |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 4 | * Copyright (C) 2019 Western Digital Corporation or its affiliates. |
Nick Kossifidis | e53d281 | 2021-04-19 03:55:38 +0300 | [diff] [blame] | 5 | * Copyright (C) 2020 FORTH-ICS/CARV |
| 6 | * Nick Kossifidis <mick@ics.forth.gr> |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 7 | */ |
| 8 | |
| 9 | #include <linux/init.h> |
| 10 | #include <linux/mm.h> |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 11 | #include <linux/memblock.h> |
Mike Rapoport | 57c8a66 | 2018-10-30 15:09:49 -0700 | [diff] [blame] | 12 | #include <linux/initrd.h> |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 13 | #include <linux/swap.h> |
Christoph Hellwig | 5ec9c4f | 2018-01-16 09:37:50 +0100 | [diff] [blame] | 14 | #include <linux/sizes.h> |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 15 | #include <linux/of_fdt.h> |
Nick Kossifidis | 5640975 | 2021-04-19 03:55:39 +0300 | [diff] [blame] | 16 | #include <linux/of_reserved_mem.h> |
Albert Ou | 922b037 | 2019-09-27 16:14:18 -0700 | [diff] [blame] | 17 | #include <linux/libfdt.h> |
Zong Li | d27c3c9 | 2020-03-10 00:55:41 +0800 | [diff] [blame] | 18 | #include <linux/set_memory.h> |
Kefeng Wang | da81558 | 2020-10-31 14:01:12 +0800 | [diff] [blame] | 19 | #include <linux/dma-map-ops.h> |
Nick Kossifidis | e53d281 | 2021-04-19 03:55:38 +0300 | [diff] [blame] | 20 | #include <linux/crash_dump.h> |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 21 | |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 22 | #include <asm/fixmap.h> |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 23 | #include <asm/tlbflush.h> |
| 24 | #include <asm/sections.h> |
Palmer Dabbelt | 2d26825 | 2020-04-14 13:43:24 +0900 | [diff] [blame] | 25 | #include <asm/soc.h> |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 26 | #include <asm/io.h> |
Zong Li | b422d28 | 2020-06-03 16:03:55 -0700 | [diff] [blame] | 27 | #include <asm/ptdump.h> |
Atish Patra | 4f0e8ee | 2020-11-18 16:38:29 -0800 | [diff] [blame] | 28 | #include <asm/numa.h> |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 29 | |
Paul Walmsley | ffaee27 | 2019-10-17 15:00:17 -0700 | [diff] [blame] | 30 | #include "../kernel/head.h" |
| 31 | |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 32 | unsigned long kernel_virt_addr = KERNEL_LINK_ADDR; |
| 33 | EXPORT_SYMBOL(kernel_virt_addr); |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 34 | #ifdef CONFIG_XIP_KERNEL |
| 35 | #define kernel_virt_addr (*((unsigned long *)XIP_FIXUP(&kernel_virt_addr))) |
| 36 | #endif |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 37 | |
Anup Patel | 387181d | 2019-03-26 08:03:47 +0000 | [diff] [blame] | 38 | unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] |
| 39 | __page_aligned_bss; |
| 40 | EXPORT_SYMBOL(empty_zero_page); |
| 41 | |
Anup Patel | d90d45d | 2019-06-07 06:01:29 +0000 | [diff] [blame] | 42 | extern char _start[]; |
Anup Patel | 8f3a2b4 | 2020-09-17 15:37:10 -0700 | [diff] [blame] | 43 | #define DTB_EARLY_BASE_VA PGDIR_SIZE |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 44 | void *_dtb_early_va __initdata; |
| 45 | uintptr_t _dtb_early_pa __initdata; |
Anup Patel | d90d45d | 2019-06-07 06:01:29 +0000 | [diff] [blame] | 46 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 47 | struct pt_alloc_ops { |
| 48 | pte_t *(*get_pte_virt)(phys_addr_t pa); |
| 49 | phys_addr_t (*alloc_pte)(uintptr_t va); |
| 50 | #ifndef __PAGETABLE_PMD_FOLDED |
| 51 | pmd_t *(*get_pmd_virt)(phys_addr_t pa); |
| 52 | phys_addr_t (*alloc_pmd)(uintptr_t va); |
| 53 | #endif |
| 54 | }; |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 55 | |
Kefeng Wang | da81558 | 2020-10-31 14:01:12 +0800 | [diff] [blame] | 56 | static phys_addr_t dma32_phys_limit __ro_after_init; |
| 57 | |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 58 | static void __init zone_sizes_init(void) |
| 59 | { |
Christoph Hellwig | 5ec9c4f | 2018-01-16 09:37:50 +0100 | [diff] [blame] | 60 | unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 61 | |
Zong Li | d5fad48 | 2018-06-25 16:49:37 +0800 | [diff] [blame] | 62 | #ifdef CONFIG_ZONE_DMA32 |
Kefeng Wang | da81558 | 2020-10-31 14:01:12 +0800 | [diff] [blame] | 63 | max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); |
Zong Li | d5fad48 | 2018-06-25 16:49:37 +0800 | [diff] [blame] | 64 | #endif |
Christoph Hellwig | 5ec9c4f | 2018-01-16 09:37:50 +0100 | [diff] [blame] | 65 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
| 66 | |
Mike Rapoport | 9691a07 | 2020-06-03 15:57:10 -0700 | [diff] [blame] | 67 | free_area_init(max_zone_pfns); |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 68 | } |
| 69 | |
Jisheng Zhang | 1987501 | 2021-03-30 02:22:21 +0800 | [diff] [blame] | 70 | static void __init setup_zero_page(void) |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 71 | { |
| 72 | memset((void *)empty_zero_page, 0, PAGE_SIZE); |
| 73 | } |
| 74 | |
Kefeng Wang | 8fa3cdf | 2020-05-14 19:53:35 +0800 | [diff] [blame] | 75 | #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) |
Yash Shah | 2cc6c4a | 2019-11-18 05:58:34 +0000 | [diff] [blame] | 76 | static inline void print_mlk(char *name, unsigned long b, unsigned long t) |
| 77 | { |
| 78 | pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, |
| 79 | (((t) - (b)) >> 10)); |
| 80 | } |
| 81 | |
| 82 | static inline void print_mlm(char *name, unsigned long b, unsigned long t) |
| 83 | { |
| 84 | pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, |
| 85 | (((t) - (b)) >> 20)); |
| 86 | } |
| 87 | |
Jisheng Zhang | 1987501 | 2021-03-30 02:22:21 +0800 | [diff] [blame] | 88 | static void __init print_vm_layout(void) |
Yash Shah | 2cc6c4a | 2019-11-18 05:58:34 +0000 | [diff] [blame] | 89 | { |
| 90 | pr_notice("Virtual kernel memory layout:\n"); |
| 91 | print_mlk("fixmap", (unsigned long)FIXADDR_START, |
| 92 | (unsigned long)FIXADDR_TOP); |
| 93 | print_mlm("pci io", (unsigned long)PCI_IO_START, |
| 94 | (unsigned long)PCI_IO_END); |
| 95 | print_mlm("vmemmap", (unsigned long)VMEMMAP_START, |
| 96 | (unsigned long)VMEMMAP_END); |
| 97 | print_mlm("vmalloc", (unsigned long)VMALLOC_START, |
| 98 | (unsigned long)VMALLOC_END); |
| 99 | print_mlm("lowmem", (unsigned long)PAGE_OFFSET, |
| 100 | (unsigned long)high_memory); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 101 | #ifdef CONFIG_64BIT |
| 102 | print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR, |
| 103 | (unsigned long)ADDRESS_SPACE_END); |
| 104 | #endif |
Yash Shah | 2cc6c4a | 2019-11-18 05:58:34 +0000 | [diff] [blame] | 105 | } |
| 106 | #else |
| 107 | static void print_vm_layout(void) { } |
| 108 | #endif /* CONFIG_DEBUG_VM */ |
| 109 | |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 110 | void __init mem_init(void) |
| 111 | { |
| 112 | #ifdef CONFIG_FLATMEM |
| 113 | BUG_ON(!mem_map); |
| 114 | #endif /* CONFIG_FLATMEM */ |
| 115 | |
| 116 | high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); |
Mike Rapoport | c6ffc5c | 2018-10-30 15:09:30 -0700 | [diff] [blame] | 117 | memblock_free_all(); |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 118 | |
Yash Shah | 2cc6c4a | 2019-11-18 05:58:34 +0000 | [diff] [blame] | 119 | print_vm_layout(); |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 120 | } |
| 121 | |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 122 | void __init setup_bootmem(void) |
| 123 | { |
Zong Li | ac51e00 | 2020-01-02 11:12:40 +0800 | [diff] [blame] | 124 | phys_addr_t vmlinux_end = __pa_symbol(&_end); |
| 125 | phys_addr_t vmlinux_start = __pa_symbol(&_start); |
Kefeng Wang | dd2d082 | 2021-02-09 09:01:51 +0800 | [diff] [blame] | 126 | phys_addr_t dram_end = memblock_end_of_DRAM(); |
Atish Patra | abb8e86 | 2021-01-11 15:45:02 -0800 | [diff] [blame] | 127 | phys_addr_t max_mapped_addr = __pa(~(ulong)0); |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 128 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 129 | #ifdef CONFIG_XIP_KERNEL |
| 130 | vmlinux_start = __pa_symbol(&_sdata); |
| 131 | #endif |
| 132 | |
Kefeng Wang | dd2d082 | 2021-02-09 09:01:51 +0800 | [diff] [blame] | 133 | /* The maximal physical memory size is -PAGE_OFFSET. */ |
Atish Patra | de043da | 2020-12-18 16:13:56 -0800 | [diff] [blame] | 134 | memblock_enforce_memory_limit(-PAGE_OFFSET); |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 135 | |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 136 | /* |
| 137 | * Reserve from the start of the kernel to the end of the kernel |
Geert Uytterhoeven | 8db6f93 | 2021-04-29 17:05:00 +0200 | [diff] [blame] | 138 | */ |
| 139 | #if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX) |
| 140 | /* |
| 141 | * Make sure we align the reservation on PMD_SIZE since we will |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 142 | * map the kernel in the linear mapping as read-only: we do not want |
| 143 | * any allocation to happen between _end and the next pmd aligned page. |
| 144 | */ |
Geert Uytterhoeven | 8db6f93 | 2021-04-29 17:05:00 +0200 | [diff] [blame] | 145 | vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; |
| 146 | #endif |
| 147 | memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); |
Anup Patel | d90d45d | 2019-06-07 06:01:29 +0000 | [diff] [blame] | 148 | |
Atish Patra | abb8e86 | 2021-01-11 15:45:02 -0800 | [diff] [blame] | 149 | /* |
| 150 | * memblock allocator is not aware of the fact that last 4K bytes of |
| 151 | * the addressable memory can not be mapped because of IS_ERR_VALUE |
| 152 | * macro. Make sure that last 4k bytes are not usable by memblock |
| 153 | * if end of dram is equal to maximum addressable memory. |
| 154 | */ |
| 155 | if (max_mapped_addr == (dram_end - 1)) |
| 156 | memblock_set_current_limit(max_mapped_addr - 4096); |
| 157 | |
Kefeng Wang | f6e5aed | 2021-02-25 14:54:17 +0800 | [diff] [blame] | 158 | min_low_pfn = PFN_UP(memblock_start_of_DRAM()); |
| 159 | max_low_pfn = max_pfn = PFN_DOWN(dram_end); |
| 160 | |
Kefeng Wang | da81558 | 2020-10-31 14:01:12 +0800 | [diff] [blame] | 161 | dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); |
Guo Ren | 336e8eb | 2021-01-21 14:31:17 +0800 | [diff] [blame] | 162 | set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 163 | |
Kefeng Wang | aec33b5 | 2021-01-15 13:46:06 +0800 | [diff] [blame] | 164 | reserve_initrd_mem(); |
Albert Ou | 922b037 | 2019-09-27 16:14:18 -0700 | [diff] [blame] | 165 | /* |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 166 | * If DTB is built in, no need to reserve its memblock. |
| 167 | * Otherwise, do reserve it but avoid using |
| 168 | * early_init_fdt_reserve_self() since __pa() does |
Albert Ou | 922b037 | 2019-09-27 16:14:18 -0700 | [diff] [blame] | 169 | * not work for DTB pointers that are fixmap addresses |
| 170 | */ |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 171 | if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) |
| 172 | memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); |
Albert Ou | 922b037 | 2019-09-27 16:14:18 -0700 | [diff] [blame] | 173 | |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 174 | early_init_fdt_scan_reserved_mem(); |
Kefeng Wang | da81558 | 2020-10-31 14:01:12 +0800 | [diff] [blame] | 175 | dma_contiguous_reserve(dma32_phys_limit); |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 176 | memblock_allow_resize(); |
Anup Patel | 0651c26 | 2019-02-21 11:25:49 +0530 | [diff] [blame] | 177 | } |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 178 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 179 | #ifdef CONFIG_XIP_KERNEL |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 180 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 181 | extern char _xiprom[], _exiprom[]; |
| 182 | extern char _sdata[], _edata[]; |
| 183 | |
| 184 | #endif /* CONFIG_XIP_KERNEL */ |
| 185 | |
Palmer Dabbelt | 76d2a04 | 2017-07-10 18:00:26 -0700 | [diff] [blame] | 186 | #ifdef CONFIG_MMU |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 187 | static struct pt_alloc_ops _pt_ops __ro_after_init; |
| 188 | |
| 189 | #ifdef CONFIG_XIP_KERNEL |
| 190 | #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&_pt_ops)) |
| 191 | #else |
| 192 | #define pt_ops _pt_ops |
| 193 | #endif |
Christoph Hellwig | 6bd33e1 | 2019-10-28 13:10:41 +0100 | [diff] [blame] | 194 | |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 195 | /* Offset between linear mapping virtual address and kernel load address */ |
Jisheng Zhang | de31ea4 | 2021-03-30 02:22:51 +0800 | [diff] [blame] | 196 | unsigned long va_pa_offset __ro_after_init; |
Anup Patel | 387181d | 2019-03-26 08:03:47 +0000 | [diff] [blame] | 197 | EXPORT_SYMBOL(va_pa_offset); |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 198 | #ifdef CONFIG_XIP_KERNEL |
| 199 | #define va_pa_offset (*((unsigned long *)XIP_FIXUP(&va_pa_offset))) |
| 200 | #endif |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 201 | /* Offset between kernel mapping virtual address and kernel load address */ |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 202 | #ifdef CONFIG_64BIT |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 203 | unsigned long va_kernel_pa_offset; |
| 204 | EXPORT_SYMBOL(va_kernel_pa_offset); |
| 205 | #endif |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 206 | #ifdef CONFIG_XIP_KERNEL |
| 207 | #define va_kernel_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_pa_offset))) |
| 208 | #endif |
| 209 | unsigned long va_kernel_xip_pa_offset; |
| 210 | EXPORT_SYMBOL(va_kernel_xip_pa_offset); |
| 211 | #ifdef CONFIG_XIP_KERNEL |
| 212 | #define va_kernel_xip_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_xip_pa_offset))) |
| 213 | #endif |
Jisheng Zhang | de31ea4 | 2021-03-30 02:22:51 +0800 | [diff] [blame] | 214 | unsigned long pfn_base __ro_after_init; |
Anup Patel | 387181d | 2019-03-26 08:03:47 +0000 | [diff] [blame] | 215 | EXPORT_SYMBOL(pfn_base); |
| 216 | |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 217 | pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 218 | pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 219 | pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 220 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 221 | pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 222 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 223 | #ifdef CONFIG_XIP_KERNEL |
| 224 | #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) |
| 225 | #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) |
| 226 | #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) |
| 227 | #endif /* CONFIG_XIP_KERNEL */ |
| 228 | |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 229 | void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) |
| 230 | { |
| 231 | unsigned long addr = __fix_to_virt(idx); |
| 232 | pte_t *ptep; |
| 233 | |
| 234 | BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); |
| 235 | |
| 236 | ptep = &fixmap_pte[pte_index(addr)]; |
| 237 | |
Greentime Hu | 21190b7 | 2020-08-04 11:02:05 +0800 | [diff] [blame] | 238 | if (pgprot_val(prot)) |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 239 | set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); |
Greentime Hu | 21190b7 | 2020-08-04 11:02:05 +0800 | [diff] [blame] | 240 | else |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 241 | pte_clear(&init_mm, addr, ptep); |
Greentime Hu | 21190b7 | 2020-08-04 11:02:05 +0800 | [diff] [blame] | 242 | local_flush_tlb_page(addr); |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 243 | } |
| 244 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 245 | static inline pte_t *__init get_pte_virt_early(phys_addr_t pa) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 246 | { |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 247 | return (pte_t *)((uintptr_t)pa); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 248 | } |
| 249 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 250 | static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) |
| 251 | { |
| 252 | clear_fixmap(FIX_PTE); |
| 253 | return (pte_t *)set_fixmap_offset(FIX_PTE, pa); |
| 254 | } |
| 255 | |
| 256 | static inline pte_t *get_pte_virt_late(phys_addr_t pa) |
| 257 | { |
| 258 | return (pte_t *) __va(pa); |
| 259 | } |
| 260 | |
| 261 | static inline phys_addr_t __init alloc_pte_early(uintptr_t va) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 262 | { |
| 263 | /* |
| 264 | * We only create PMD or PGD early mappings so we |
| 265 | * should never reach here with MMU disabled. |
| 266 | */ |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 267 | BUG(); |
| 268 | } |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 269 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 270 | static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) |
| 271 | { |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 272 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); |
| 273 | } |
| 274 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 275 | static phys_addr_t alloc_pte_late(uintptr_t va) |
| 276 | { |
| 277 | unsigned long vaddr; |
| 278 | |
| 279 | vaddr = __get_free_page(GFP_KERNEL); |
zhouchuangao | e75e6bf | 2021-03-30 06:56:26 -0700 | [diff] [blame] | 280 | BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr))); |
| 281 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 282 | return __pa(vaddr); |
| 283 | } |
| 284 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 285 | static void __init create_pte_mapping(pte_t *ptep, |
| 286 | uintptr_t va, phys_addr_t pa, |
| 287 | phys_addr_t sz, pgprot_t prot) |
| 288 | { |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 289 | uintptr_t pte_idx = pte_index(va); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 290 | |
| 291 | BUG_ON(sz != PAGE_SIZE); |
| 292 | |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 293 | if (pte_none(ptep[pte_idx])) |
| 294 | ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | #ifndef __PAGETABLE_PMD_FOLDED |
| 298 | |
| 299 | pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; |
| 300 | pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; |
Alexandre Ghiti | 0f02de4 | 2021-02-21 09:22:33 -0500 | [diff] [blame] | 301 | pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); |
Anup Patel | 1074dd4 | 2020-11-04 12:07:13 +0530 | [diff] [blame] | 302 | pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 303 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 304 | #ifdef CONFIG_XIP_KERNEL |
| 305 | #define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) |
| 306 | #define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) |
| 307 | #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) |
| 308 | #endif /* CONFIG_XIP_KERNEL */ |
| 309 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 310 | static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 311 | { |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 312 | /* Before MMU is enabled */ |
| 313 | return (pmd_t *)((uintptr_t)pa); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 314 | } |
| 315 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 316 | static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) |
| 317 | { |
| 318 | clear_fixmap(FIX_PMD); |
| 319 | return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); |
| 320 | } |
| 321 | |
| 322 | static pmd_t *get_pmd_virt_late(phys_addr_t pa) |
| 323 | { |
| 324 | return (pmd_t *) __va(pa); |
| 325 | } |
| 326 | |
| 327 | static phys_addr_t __init alloc_pmd_early(uintptr_t va) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 328 | { |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 329 | BUG_ON((va - kernel_virt_addr) >> PGDIR_SHIFT); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 330 | |
Alexandre Ghiti | 0f02de4 | 2021-02-21 09:22:33 -0500 | [diff] [blame] | 331 | return (uintptr_t)early_pmd; |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 332 | } |
| 333 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 334 | static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) |
| 335 | { |
| 336 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); |
| 337 | } |
| 338 | |
| 339 | static phys_addr_t alloc_pmd_late(uintptr_t va) |
| 340 | { |
| 341 | unsigned long vaddr; |
| 342 | |
| 343 | vaddr = __get_free_page(GFP_KERNEL); |
| 344 | BUG_ON(!vaddr); |
| 345 | return __pa(vaddr); |
| 346 | } |
| 347 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 348 | static void __init create_pmd_mapping(pmd_t *pmdp, |
| 349 | uintptr_t va, phys_addr_t pa, |
| 350 | phys_addr_t sz, pgprot_t prot) |
| 351 | { |
| 352 | pte_t *ptep; |
| 353 | phys_addr_t pte_phys; |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 354 | uintptr_t pmd_idx = pmd_index(va); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 355 | |
| 356 | if (sz == PMD_SIZE) { |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 357 | if (pmd_none(pmdp[pmd_idx])) |
| 358 | pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 359 | return; |
| 360 | } |
| 361 | |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 362 | if (pmd_none(pmdp[pmd_idx])) { |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 363 | pte_phys = pt_ops.alloc_pte(va); |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 364 | pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 365 | ptep = pt_ops.get_pte_virt(pte_phys); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 366 | memset(ptep, 0, PAGE_SIZE); |
| 367 | } else { |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 368 | pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 369 | ptep = pt_ops.get_pte_virt(pte_phys); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 370 | } |
| 371 | |
| 372 | create_pte_mapping(ptep, va, pa, sz, prot); |
| 373 | } |
| 374 | |
| 375 | #define pgd_next_t pmd_t |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 376 | #define alloc_pgd_next(__va) pt_ops.alloc_pmd(__va) |
| 377 | #define get_pgd_next_virt(__pa) pt_ops.get_pmd_virt(__pa) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 378 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ |
| 379 | create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 380 | #define fixmap_pgd_next fixmap_pmd |
| 381 | #else |
| 382 | #define pgd_next_t pte_t |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 383 | #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) |
| 384 | #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 385 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ |
| 386 | create_pte_mapping(__nextp, __va, __pa, __sz, __prot) |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 387 | #define fixmap_pgd_next fixmap_pte |
| 388 | #endif |
| 389 | |
Atish Patra | b91540d | 2020-09-17 15:37:15 -0700 | [diff] [blame] | 390 | void __init create_pgd_mapping(pgd_t *pgdp, |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 391 | uintptr_t va, phys_addr_t pa, |
| 392 | phys_addr_t sz, pgprot_t prot) |
| 393 | { |
| 394 | pgd_next_t *nextp; |
| 395 | phys_addr_t next_phys; |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 396 | uintptr_t pgd_idx = pgd_index(va); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 397 | |
| 398 | if (sz == PGDIR_SIZE) { |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 399 | if (pgd_val(pgdp[pgd_idx]) == 0) |
| 400 | pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 401 | return; |
| 402 | } |
| 403 | |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 404 | if (pgd_val(pgdp[pgd_idx]) == 0) { |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 405 | next_phys = alloc_pgd_next(va); |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 406 | pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 407 | nextp = get_pgd_next_virt(next_phys); |
| 408 | memset(nextp, 0, PAGE_SIZE); |
| 409 | } else { |
Mike Rapoport | 974b9b2 | 2020-06-08 21:33:10 -0700 | [diff] [blame] | 410 | next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 411 | nextp = get_pgd_next_virt(next_phys); |
| 412 | } |
| 413 | |
| 414 | create_pgd_next_mapping(nextp, va, pa, sz, prot); |
| 415 | } |
| 416 | |
| 417 | static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) |
| 418 | { |
Zong Li | 0fdc636 | 2019-11-08 01:00:40 -0800 | [diff] [blame] | 419 | /* Upgrade to PMD_SIZE mappings whenever possible */ |
| 420 | if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) |
| 421 | return PAGE_SIZE; |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 422 | |
Zong Li | 0fdc636 | 2019-11-08 01:00:40 -0800 | [diff] [blame] | 423 | return PMD_SIZE; |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 424 | } |
| 425 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 426 | #ifdef CONFIG_XIP_KERNEL |
| 427 | /* called from head.S with MMU off */ |
| 428 | asmlinkage void __init __copy_data(void) |
| 429 | { |
| 430 | void *from = (void *)(&_sdata); |
| 431 | void *end = (void *)(&_end); |
| 432 | void *to = (void *)CONFIG_PHYS_RAM_BASE; |
| 433 | size_t sz = (size_t)(end - from + 1); |
| 434 | |
| 435 | memcpy(to, from, sz); |
| 436 | } |
| 437 | #endif |
| 438 | |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 439 | #ifdef CONFIG_STRICT_KERNEL_RWX |
| 440 | static __init pgprot_t pgprot_from_va(uintptr_t va) |
| 441 | { |
| 442 | if (is_va_kernel_text(va)) |
| 443 | return PAGE_KERNEL_READ_EXEC; |
| 444 | |
| 445 | /* |
| 446 | * In 64-bit kernel, the kernel mapping is outside the linear mapping so |
| 447 | * we must protect its linear mapping alias from being executed and |
| 448 | * written. |
| 449 | * And rodata section is marked readonly in mark_rodata_ro. |
| 450 | */ |
| 451 | if (IS_ENABLED(CONFIG_64BIT) && is_va_kernel_lm_alias_text(va)) |
| 452 | return PAGE_KERNEL_READ; |
| 453 | |
| 454 | return PAGE_KERNEL; |
| 455 | } |
| 456 | |
| 457 | void mark_rodata_ro(void) |
| 458 | { |
| 459 | set_kernel_memory(__start_rodata, _data, set_memory_ro); |
| 460 | if (IS_ENABLED(CONFIG_64BIT)) |
| 461 | set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data), |
| 462 | set_memory_ro); |
| 463 | |
| 464 | debug_checkwx(); |
| 465 | } |
| 466 | #else |
| 467 | static __init pgprot_t pgprot_from_va(uintptr_t va) |
| 468 | { |
| 469 | if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va)) |
| 470 | return PAGE_KERNEL; |
| 471 | |
| 472 | return PAGE_KERNEL_EXEC; |
| 473 | } |
| 474 | #endif /* CONFIG_STRICT_KERNEL_RWX */ |
| 475 | |
Anup Patel | 387181d | 2019-03-26 08:03:47 +0000 | [diff] [blame] | 476 | /* |
| 477 | * setup_vm() is called from head.S with MMU-off. |
| 478 | * |
| 479 | * Following requirements should be honoured for setup_vm() to work |
| 480 | * correctly: |
| 481 | * 1) It should use PC-relative addressing for accessing kernel symbols. |
| 482 | * To achieve this we always use GCC cmodel=medany. |
| 483 | * 2) The compiler instrumentation for FTRACE will not work for setup_vm() |
| 484 | * so disable compiler instrumentation when FTRACE is enabled. |
| 485 | * |
| 486 | * Currently, the above requirements are honoured by using custom CFLAGS |
| 487 | * for init.o in mm/Makefile. |
| 488 | */ |
| 489 | |
| 490 | #ifndef __riscv_cmodel_medany |
Paul Walmsley | 6a527b6 | 2019-10-17 14:45:58 -0700 | [diff] [blame] | 491 | #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." |
Anup Patel | 387181d | 2019-03-26 08:03:47 +0000 | [diff] [blame] | 492 | #endif |
| 493 | |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 494 | static uintptr_t load_pa __initdata; |
| 495 | uintptr_t load_sz; |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 496 | #ifdef CONFIG_XIP_KERNEL |
| 497 | #define load_pa (*((uintptr_t *)XIP_FIXUP(&load_pa))) |
| 498 | #define load_sz (*((uintptr_t *)XIP_FIXUP(&load_sz))) |
| 499 | #endif |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 500 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 501 | #ifdef CONFIG_XIP_KERNEL |
| 502 | uintptr_t xiprom, xiprom_sz; |
| 503 | #define xiprom_sz (*((uintptr_t *)XIP_FIXUP(&xiprom_sz))) |
| 504 | #define xiprom (*((uintptr_t *)XIP_FIXUP(&xiprom))) |
| 505 | |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 506 | static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size, |
| 507 | __always_unused bool early) |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 508 | { |
| 509 | uintptr_t va, end_va; |
| 510 | |
| 511 | /* Map the flash resident part */ |
| 512 | end_va = kernel_virt_addr + xiprom_sz; |
| 513 | for (va = kernel_virt_addr; va < end_va; va += map_size) |
| 514 | create_pgd_mapping(pgdir, va, |
| 515 | xiprom + (va - kernel_virt_addr), |
| 516 | map_size, PAGE_KERNEL_EXEC); |
| 517 | |
| 518 | /* Map the data in RAM */ |
| 519 | end_va = kernel_virt_addr + XIP_OFFSET + load_sz; |
| 520 | for (va = kernel_virt_addr + XIP_OFFSET; va < end_va; va += map_size) |
| 521 | create_pgd_mapping(pgdir, va, |
| 522 | load_pa + (va - (kernel_virt_addr + XIP_OFFSET)), |
| 523 | map_size, PAGE_KERNEL); |
| 524 | } |
| 525 | #else |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 526 | static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size, |
| 527 | bool early) |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 528 | { |
| 529 | uintptr_t va, end_va; |
| 530 | |
| 531 | end_va = kernel_virt_addr + load_sz; |
| 532 | for (va = kernel_virt_addr; va < end_va; va += map_size) |
| 533 | create_pgd_mapping(pgdir, va, |
| 534 | load_pa + (va - kernel_virt_addr), |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 535 | map_size, |
| 536 | early ? |
| 537 | PAGE_KERNEL_EXEC : pgprot_from_va(va)); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 538 | } |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 539 | #endif |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 540 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 541 | asmlinkage void __init setup_vm(uintptr_t dtb_pa) |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 542 | { |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 543 | uintptr_t __maybe_unused pa; |
Alexandre Ghiti | 0f02de4 | 2021-02-21 09:22:33 -0500 | [diff] [blame] | 544 | uintptr_t map_size; |
Atish Patra | 6262f66 | 2020-09-17 15:37:11 -0700 | [diff] [blame] | 545 | #ifndef __PAGETABLE_PMD_FOLDED |
| 546 | pmd_t fix_bmap_spmd, fix_bmap_epmd; |
| 547 | #endif |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 548 | |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 549 | #ifdef CONFIG_XIP_KERNEL |
| 550 | xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; |
| 551 | xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); |
| 552 | |
| 553 | load_pa = (uintptr_t)CONFIG_PHYS_RAM_BASE; |
| 554 | load_sz = (uintptr_t)(&_end) - (uintptr_t)(&_sdata); |
| 555 | |
| 556 | va_kernel_xip_pa_offset = kernel_virt_addr - xiprom; |
| 557 | #else |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 558 | load_pa = (uintptr_t)(&_start); |
| 559 | load_sz = (uintptr_t)(&_end) - load_pa; |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 560 | #endif |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 561 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 562 | va_pa_offset = PAGE_OFFSET - load_pa; |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 563 | #ifdef CONFIG_64BIT |
| 564 | va_kernel_pa_offset = kernel_virt_addr - load_pa; |
| 565 | #endif |
| 566 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 567 | pfn_base = PFN_DOWN(load_pa); |
| 568 | |
| 569 | /* |
| 570 | * Enforce boot alignment requirements of RV32 and |
| 571 | * RV64 by only allowing PMD or PGD mappings. |
| 572 | */ |
Alexandre Ghiti | 0f02de4 | 2021-02-21 09:22:33 -0500 | [diff] [blame] | 573 | map_size = PMD_SIZE; |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 574 | |
| 575 | /* Sanity check alignment and size */ |
| 576 | BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 577 | BUG_ON((load_pa % map_size) != 0); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 578 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 579 | pt_ops.alloc_pte = alloc_pte_early; |
| 580 | pt_ops.get_pte_virt = get_pte_virt_early; |
| 581 | #ifndef __PAGETABLE_PMD_FOLDED |
| 582 | pt_ops.alloc_pmd = alloc_pmd_early; |
| 583 | pt_ops.get_pmd_virt = get_pmd_virt_early; |
| 584 | #endif |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 585 | /* Setup early PGD for fixmap */ |
| 586 | create_pgd_mapping(early_pg_dir, FIXADDR_START, |
| 587 | (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 588 | |
| 589 | #ifndef __PAGETABLE_PMD_FOLDED |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 590 | /* Setup fixmap PMD */ |
| 591 | create_pmd_mapping(fixmap_pmd, FIXADDR_START, |
| 592 | (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); |
| 593 | /* Setup trampoline PGD and PMD */ |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 594 | create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 595 | (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 596 | #ifdef CONFIG_XIP_KERNEL |
| 597 | create_pmd_mapping(trampoline_pmd, kernel_virt_addr, |
| 598 | xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); |
| 599 | #else |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 600 | create_pmd_mapping(trampoline_pmd, kernel_virt_addr, |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 601 | load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 602 | #endif |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 603 | #else |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 604 | /* Setup trampoline PGD */ |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 605 | create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 606 | load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); |
| 607 | #endif |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 608 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 609 | /* |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 610 | * Setup early PGD covering entire kernel which will allow |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 611 | * us to reach paging_init(). We map all memory banks later |
| 612 | * in setup_vm_final() below. |
| 613 | */ |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 614 | create_kernel_page_table(early_pg_dir, map_size, true); |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 615 | |
Anup Patel | 1074dd4 | 2020-11-04 12:07:13 +0530 | [diff] [blame] | 616 | #ifndef __PAGETABLE_PMD_FOLDED |
| 617 | /* Setup early PMD for DTB */ |
| 618 | create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, |
| 619 | (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE); |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 620 | #ifndef CONFIG_BUILTIN_DTB |
Anup Patel | 1074dd4 | 2020-11-04 12:07:13 +0530 | [diff] [blame] | 621 | /* Create two consecutive PMD mappings for FDT early scan */ |
| 622 | pa = dtb_pa & ~(PMD_SIZE - 1); |
| 623 | create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, |
| 624 | pa, PMD_SIZE, PAGE_KERNEL); |
| 625 | create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, |
| 626 | pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); |
| 627 | dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 628 | #else /* CONFIG_BUILTIN_DTB */ |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 629 | #ifdef CONFIG_64BIT |
| 630 | /* |
| 631 | * __va can't be used since it would return a linear mapping address |
| 632 | * whereas dtb_early_va will be used before setup_vm_final installs |
| 633 | * the linear mapping. |
| 634 | */ |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 635 | dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 636 | #else |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 637 | dtb_early_va = __va(dtb_pa); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 638 | #endif /* CONFIG_64BIT */ |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 639 | #endif /* CONFIG_BUILTIN_DTB */ |
Anup Patel | 1074dd4 | 2020-11-04 12:07:13 +0530 | [diff] [blame] | 640 | #else |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 641 | #ifndef CONFIG_BUILTIN_DTB |
Anup Patel | 8f3a2b4 | 2020-09-17 15:37:10 -0700 | [diff] [blame] | 642 | /* Create two consecutive PGD mappings for FDT early scan */ |
| 643 | pa = dtb_pa & ~(PGDIR_SIZE - 1); |
| 644 | create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, |
| 645 | pa, PGDIR_SIZE, PAGE_KERNEL); |
| 646 | create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE, |
| 647 | pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL); |
| 648 | dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1)); |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 649 | #else /* CONFIG_BUILTIN_DTB */ |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 650 | #ifdef CONFIG_64BIT |
Vitaly Wool | 44c9225 | 2021-04-13 02:35:14 -0400 | [diff] [blame] | 651 | dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 652 | #else |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 653 | dtb_early_va = __va(dtb_pa); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 654 | #endif /* CONFIG_64BIT */ |
Vitaly Wool | f105aa9 | 2021-01-16 01:49:48 +0200 | [diff] [blame] | 655 | #endif /* CONFIG_BUILTIN_DTB */ |
Anup Patel | 1074dd4 | 2020-11-04 12:07:13 +0530 | [diff] [blame] | 656 | #endif |
Albert Ou | 922b037 | 2019-09-27 16:14:18 -0700 | [diff] [blame] | 657 | dtb_early_pa = dtb_pa; |
Atish Patra | 6262f66 | 2020-09-17 15:37:11 -0700 | [diff] [blame] | 658 | |
| 659 | /* |
| 660 | * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap |
| 661 | * range can not span multiple pmds. |
| 662 | */ |
| 663 | BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) |
| 664 | != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); |
| 665 | |
| 666 | #ifndef __PAGETABLE_PMD_FOLDED |
| 667 | /* |
| 668 | * Early ioremap fixmap is already created as it lies within first 2MB |
| 669 | * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END |
| 670 | * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn |
| 671 | * the user if not. |
| 672 | */ |
| 673 | fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))]; |
| 674 | fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))]; |
| 675 | if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) { |
| 676 | WARN_ON(1); |
| 677 | pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n", |
| 678 | pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd)); |
| 679 | pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", |
| 680 | fix_to_virt(FIX_BTMAP_BEGIN)); |
| 681 | pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", |
| 682 | fix_to_virt(FIX_BTMAP_END)); |
| 683 | |
| 684 | pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); |
| 685 | pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); |
| 686 | } |
| 687 | #endif |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 688 | } |
| 689 | |
| 690 | static void __init setup_vm_final(void) |
| 691 | { |
| 692 | uintptr_t va, map_size; |
| 693 | phys_addr_t pa, start, end; |
Mike Rapoport | b10d6bc | 2020-10-13 16:58:08 -0700 | [diff] [blame] | 694 | u64 i; |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 695 | |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 696 | /** |
| 697 | * MMU is enabled at this point. But page table setup is not complete yet. |
| 698 | * fixmap page table alloc functions should be used at this point |
| 699 | */ |
| 700 | pt_ops.alloc_pte = alloc_pte_fixmap; |
| 701 | pt_ops.get_pte_virt = get_pte_virt_fixmap; |
| 702 | #ifndef __PAGETABLE_PMD_FOLDED |
| 703 | pt_ops.alloc_pmd = alloc_pmd_fixmap; |
| 704 | pt_ops.get_pmd_virt = get_pmd_virt_fixmap; |
| 705 | #endif |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 706 | /* Setup swapper PGD for fixmap */ |
| 707 | create_pgd_mapping(swapper_pg_dir, FIXADDR_START, |
Zong Li | ac51e00 | 2020-01-02 11:12:40 +0800 | [diff] [blame] | 708 | __pa_symbol(fixmap_pgd_next), |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 709 | PGDIR_SIZE, PAGE_TABLE); |
| 710 | |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 711 | /* Map all memory banks in the linear mapping */ |
Mike Rapoport | b10d6bc | 2020-10-13 16:58:08 -0700 | [diff] [blame] | 712 | for_each_mem_range(i, &start, &end) { |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 713 | if (start >= end) |
| 714 | break; |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 715 | if (start <= __pa(PAGE_OFFSET) && |
| 716 | __pa(PAGE_OFFSET) < end) |
| 717 | start = __pa(PAGE_OFFSET); |
| 718 | |
| 719 | map_size = best_map_size(start, end - start); |
| 720 | for (pa = start; pa < end; pa += map_size) { |
| 721 | va = (uintptr_t)__va(pa); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 722 | |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 723 | create_pgd_mapping(swapper_pg_dir, va, pa, map_size, |
| 724 | pgprot_from_va(va)); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 725 | } |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 726 | } |
Anup Patel | f2c17aa | 2019-01-07 20:57:01 +0530 | [diff] [blame] | 727 | |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 728 | #ifdef CONFIG_64BIT |
| 729 | /* Map the kernel */ |
Alexandre Ghiti | e5c35fa0 | 2021-06-24 14:00:41 +0200 | [diff] [blame^] | 730 | create_kernel_page_table(swapper_pg_dir, PMD_SIZE, false); |
Alexandre Ghiti | 2bfc6cd | 2021-04-11 12:41:44 -0400 | [diff] [blame] | 731 | #endif |
| 732 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 733 | /* Clear fixmap PTE and PMD mappings */ |
| 734 | clear_fixmap(FIX_PTE); |
| 735 | clear_fixmap(FIX_PMD); |
| 736 | |
| 737 | /* Move to swapper page table */ |
Zong Li | ac51e00 | 2020-01-02 11:12:40 +0800 | [diff] [blame] | 738 | csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 739 | local_flush_tlb_all(); |
Atish Patra | e8dcb61 | 2020-09-17 15:37:12 -0700 | [diff] [blame] | 740 | |
| 741 | /* generic page allocation functions must be used to setup page table */ |
| 742 | pt_ops.alloc_pte = alloc_pte_late; |
| 743 | pt_ops.get_pte_virt = get_pte_virt_late; |
| 744 | #ifndef __PAGETABLE_PMD_FOLDED |
| 745 | pt_ops.alloc_pmd = alloc_pmd_late; |
| 746 | pt_ops.get_pmd_virt = get_pmd_virt_late; |
| 747 | #endif |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 748 | } |
Christoph Hellwig | 6bd33e1 | 2019-10-28 13:10:41 +0100 | [diff] [blame] | 749 | #else |
| 750 | asmlinkage void __init setup_vm(uintptr_t dtb_pa) |
| 751 | { |
| 752 | dtb_early_va = (void *)dtb_pa; |
Atish Patra | a78c6f5 | 2020-10-01 12:04:56 -0700 | [diff] [blame] | 753 | dtb_early_pa = dtb_pa; |
Christoph Hellwig | 6bd33e1 | 2019-10-28 13:10:41 +0100 | [diff] [blame] | 754 | } |
| 755 | |
| 756 | static inline void setup_vm_final(void) |
| 757 | { |
| 758 | } |
| 759 | #endif /* CONFIG_MMU */ |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 760 | |
Nick Kossifidis | e53d281 | 2021-04-19 03:55:38 +0300 | [diff] [blame] | 761 | #ifdef CONFIG_KEXEC_CORE |
| 762 | /* |
| 763 | * reserve_crashkernel() - reserves memory for crash kernel |
| 764 | * |
| 765 | * This function reserves memory area given in "crashkernel=" kernel command |
| 766 | * line parameter. The memory reserved is used by dump capture kernel when |
| 767 | * primary kernel is crashing. |
| 768 | */ |
| 769 | static void __init reserve_crashkernel(void) |
| 770 | { |
| 771 | unsigned long long crash_base = 0; |
| 772 | unsigned long long crash_size = 0; |
| 773 | unsigned long search_start = memblock_start_of_DRAM(); |
| 774 | unsigned long search_end = memblock_end_of_DRAM(); |
| 775 | |
| 776 | int ret = 0; |
| 777 | |
Nick Kossifidis | 5640975 | 2021-04-19 03:55:39 +0300 | [diff] [blame] | 778 | /* |
| 779 | * Don't reserve a region for a crash kernel on a crash kernel |
| 780 | * since it doesn't make much sense and we have limited memory |
| 781 | * resources. |
| 782 | */ |
| 783 | #ifdef CONFIG_CRASH_DUMP |
| 784 | if (is_kdump_kernel()) { |
| 785 | pr_info("crashkernel: ignoring reservation request\n"); |
| 786 | return; |
| 787 | } |
| 788 | #endif |
| 789 | |
Nick Kossifidis | e53d281 | 2021-04-19 03:55:38 +0300 | [diff] [blame] | 790 | ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), |
| 791 | &crash_size, &crash_base); |
| 792 | if (ret || !crash_size) |
| 793 | return; |
| 794 | |
| 795 | crash_size = PAGE_ALIGN(crash_size); |
| 796 | |
| 797 | if (crash_base == 0) { |
| 798 | /* |
| 799 | * Current riscv boot protocol requires 2MB alignment for |
| 800 | * RV64 and 4MB alignment for RV32 (hugepage size) |
| 801 | */ |
| 802 | crash_base = memblock_find_in_range(search_start, search_end, |
| 803 | crash_size, PMD_SIZE); |
| 804 | |
| 805 | if (crash_base == 0) { |
| 806 | pr_warn("crashkernel: couldn't allocate %lldKB\n", |
| 807 | crash_size >> 10); |
| 808 | return; |
| 809 | } |
| 810 | } else { |
| 811 | /* User specifies base address explicitly. */ |
| 812 | if (!memblock_is_region_memory(crash_base, crash_size)) { |
| 813 | pr_warn("crashkernel: requested region is not memory\n"); |
| 814 | return; |
| 815 | } |
| 816 | |
| 817 | if (memblock_is_region_reserved(crash_base, crash_size)) { |
| 818 | pr_warn("crashkernel: requested region is reserved\n"); |
| 819 | return; |
| 820 | } |
| 821 | |
| 822 | |
| 823 | if (!IS_ALIGNED(crash_base, PMD_SIZE)) { |
| 824 | pr_warn("crashkernel: requested region is misaligned\n"); |
| 825 | return; |
| 826 | } |
| 827 | } |
| 828 | memblock_reserve(crash_base, crash_size); |
| 829 | |
| 830 | pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n", |
| 831 | crash_base, crash_base + crash_size, crash_size >> 20); |
| 832 | |
| 833 | crashk_res.start = crash_base; |
| 834 | crashk_res.end = crash_base + crash_size - 1; |
| 835 | } |
| 836 | #endif /* CONFIG_KEXEC_CORE */ |
| 837 | |
Nick Kossifidis | 5640975 | 2021-04-19 03:55:39 +0300 | [diff] [blame] | 838 | #ifdef CONFIG_CRASH_DUMP |
| 839 | /* |
| 840 | * We keep track of the ELF core header of the crashed |
| 841 | * kernel with a reserved-memory region with compatible |
| 842 | * string "linux,elfcorehdr". Here we register a callback |
| 843 | * to populate elfcorehdr_addr/size when this region is |
| 844 | * present. Note that this region will be marked as |
| 845 | * reserved once we call early_init_fdt_scan_reserved_mem() |
| 846 | * later on. |
| 847 | */ |
| 848 | static int elfcore_hdr_setup(struct reserved_mem *rmem) |
| 849 | { |
| 850 | elfcorehdr_addr = rmem->base; |
| 851 | elfcorehdr_size = rmem->size; |
| 852 | return 0; |
| 853 | } |
| 854 | |
| 855 | RESERVEDMEM_OF_DECLARE(elfcorehdr, "linux,elfcorehdr", elfcore_hdr_setup); |
| 856 | #endif |
| 857 | |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 858 | void __init paging_init(void) |
| 859 | { |
| 860 | setup_vm_final(); |
| 861 | setup_zero_page(); |
Atish Patra | cbd34f4 | 2020-11-18 16:38:27 -0800 | [diff] [blame] | 862 | } |
| 863 | |
| 864 | void __init misc_mem_init(void) |
| 865 | { |
Kefeng Wang | f6e5aed | 2021-02-25 14:54:17 +0800 | [diff] [blame] | 866 | early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); |
Atish Patra | 4f0e8ee | 2020-11-18 16:38:29 -0800 | [diff] [blame] | 867 | arch_numa_init(); |
Atish Patra | cbd34f4 | 2020-11-18 16:38:27 -0800 | [diff] [blame] | 868 | sparse_init(); |
Anup Patel | 671f9a3 | 2019-06-28 13:36:21 -0700 | [diff] [blame] | 869 | zone_sizes_init(); |
Nick Kossifidis | e53d281 | 2021-04-19 03:55:38 +0300 | [diff] [blame] | 870 | #ifdef CONFIG_KEXEC_CORE |
| 871 | reserve_crashkernel(); |
| 872 | #endif |
Atish Patra | 4f0e8ee | 2020-11-18 16:38:29 -0800 | [diff] [blame] | 873 | memblock_dump_all(); |
Anup Patel | 6f1e9e9 | 2019-02-13 16:38:36 +0530 | [diff] [blame] | 874 | } |
Logan Gunthorpe | d95f1a5 | 2019-08-28 15:40:54 -0600 | [diff] [blame] | 875 | |
Kefeng Wang | 9fe57d8 | 2019-10-23 11:23:02 +0800 | [diff] [blame] | 876 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
Logan Gunthorpe | d95f1a5 | 2019-08-28 15:40:54 -0600 | [diff] [blame] | 877 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, |
| 878 | struct vmem_altmap *altmap) |
| 879 | { |
Anshuman Khandual | 1d9cfee | 2020-08-06 23:23:19 -0700 | [diff] [blame] | 880 | return vmemmap_populate_basepages(start, end, node, NULL); |
Logan Gunthorpe | d95f1a5 | 2019-08-28 15:40:54 -0600 | [diff] [blame] | 881 | } |
| 882 | #endif |