blob: adab1595f4bd89ba0729db70dd21d619d93e829d [file] [log] [blame]
Kees Cookcb18ef02016-05-09 13:22:05 -07001/*
2 * Helper routines for building identity mapping page tables. This is
3 * included by both the compressed kernel and the regular kernel.
4 */
Yinghai Lucf4fb152016-05-06 15:01:34 -07005
Rafael J. Wysockie4630fd2016-08-08 15:31:31 +02006static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
Yinghai Lucf4fb152016-05-06 15:01:34 -07007 unsigned long addr, unsigned long end)
8{
9 addr &= PMD_MASK;
10 for (; addr < end; addr += PMD_SIZE) {
11 pmd_t *pmd = pmd_page + pmd_index(addr);
12
Rafael J. Wysockie4630fd2016-08-08 15:31:31 +020013 if (pmd_present(*pmd))
14 continue;
15
Xunlei Pang66aad4f2017-05-04 09:42:50 +080016 set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
Yinghai Lucf4fb152016-05-06 15:01:34 -070017 }
18}
Kees Cookcb18ef02016-05-09 13:22:05 -070019
Yinghai Lucf4fb152016-05-06 15:01:34 -070020static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
21 unsigned long addr, unsigned long end)
22{
23 unsigned long next;
24
25 for (; addr < end; addr = next) {
26 pud_t *pud = pud_page + pud_index(addr);
27 pmd_t *pmd;
28
29 next = (addr & PUD_MASK) + PUD_SIZE;
30 if (next > end)
31 next = end;
32
Xunlei Pang66aad4f2017-05-04 09:42:50 +080033 if (info->direct_gbpages) {
34 pud_t pudval;
35
36 if (pud_present(*pud))
37 continue;
38
39 addr &= PUD_MASK;
40 pudval = __pud((addr - info->offset) | info->page_flag);
41 set_pud(pud, pudval);
42 continue;
43 }
44
Yinghai Lucf4fb152016-05-06 15:01:34 -070045 if (pud_present(*pud)) {
46 pmd = pmd_offset(pud, 0);
Rafael J. Wysockie4630fd2016-08-08 15:31:31 +020047 ident_pmd_init(info, pmd, addr, next);
Yinghai Lucf4fb152016-05-06 15:01:34 -070048 continue;
49 }
50 pmd = (pmd_t *)info->alloc_pgt_page(info->context);
51 if (!pmd)
52 return -ENOMEM;
Rafael J. Wysockie4630fd2016-08-08 15:31:31 +020053 ident_pmd_init(info, pmd, addr, next);
Yinghai Lucf4fb152016-05-06 15:01:34 -070054 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
55 }
56
57 return 0;
58}
59
Kirill A. Shutemovea3b5e62017-03-13 17:33:07 +030060static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
61 unsigned long addr, unsigned long end)
62{
63 unsigned long next;
64
65 for (; addr < end; addr = next) {
66 p4d_t *p4d = p4d_page + p4d_index(addr);
67 pud_t *pud;
68
69 next = (addr & P4D_MASK) + P4D_SIZE;
70 if (next > end)
71 next = end;
72
73 if (p4d_present(*p4d)) {
74 pud = pud_offset(p4d, 0);
75 ident_pud_init(info, pud, addr, next);
76 continue;
77 }
78 pud = (pud_t *)info->alloc_pgt_page(info->context);
79 if (!pud)
80 return -ENOMEM;
81 ident_pud_init(info, pud, addr, next);
82 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
83 }
84
85 return 0;
86}
87
Yinghai Lucf4fb152016-05-06 15:01:34 -070088int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
Rafael J. Wysockie4630fd2016-08-08 15:31:31 +020089 unsigned long pstart, unsigned long pend)
Yinghai Lucf4fb152016-05-06 15:01:34 -070090{
Rafael J. Wysockie4630fd2016-08-08 15:31:31 +020091 unsigned long addr = pstart + info->offset;
92 unsigned long end = pend + info->offset;
Yinghai Lucf4fb152016-05-06 15:01:34 -070093 unsigned long next;
94 int result;
Yinghai Lucf4fb152016-05-06 15:01:34 -070095
96 for (; addr < end; addr = next) {
Rafael J. Wysockie4630fd2016-08-08 15:31:31 +020097 pgd_t *pgd = pgd_page + pgd_index(addr);
Kirill A. Shutemovea3b5e62017-03-13 17:33:07 +030098 p4d_t *p4d;
Yinghai Lucf4fb152016-05-06 15:01:34 -070099
100 next = (addr & PGDIR_MASK) + PGDIR_SIZE;
101 if (next > end)
102 next = end;
103
104 if (pgd_present(*pgd)) {
Kirill A. Shutemovea3b5e62017-03-13 17:33:07 +0300105 p4d = p4d_offset(pgd, 0);
106 result = ident_p4d_init(info, p4d, addr, next);
Yinghai Lucf4fb152016-05-06 15:01:34 -0700107 if (result)
108 return result;
109 continue;
110 }
111
Kirill A. Shutemovea3b5e62017-03-13 17:33:07 +0300112 p4d = (p4d_t *)info->alloc_pgt_page(info->context);
113 if (!p4d)
Yinghai Lucf4fb152016-05-06 15:01:34 -0700114 return -ENOMEM;
Kirill A. Shutemovea3b5e62017-03-13 17:33:07 +0300115 result = ident_p4d_init(info, p4d, addr, next);
Yinghai Lucf4fb152016-05-06 15:01:34 -0700116 if (result)
117 return result;
Kirill A. Shutemovea3b5e62017-03-13 17:33:07 +0300118 if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
119 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
120 } else {
121 /*
122 * With p4d folded, pgd is equal to p4d.
123 * The pgd entry has to point to the pud page table in this case.
124 */
125 pud_t *pud = pud_offset(p4d, 0);
126 set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
127 }
Yinghai Lucf4fb152016-05-06 15:01:34 -0700128 }
129
130 return 0;
131}