Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef _ASM_PGALLOC_H |
| 2 | #define _ASM_PGALLOC_H |
| 3 | |
| 4 | #include <linux/gfp.h> |
| 5 | #include <linux/mm.h> |
| 6 | #include <linux/threads.h> |
| 7 | #include <asm/processor.h> |
| 8 | #include <asm/fixmap.h> |
| 9 | |
| 10 | #include <asm/cache.h> |
| 11 | |
| 12 | /* Allocate the top level pgd (page directory) |
| 13 | * |
| 14 | * Here (for 64 bit kernels) we implement a Hybrid L2/L3 scheme: we |
| 15 | * allocate the first pmd adjacent to the pgd. This means that we can |
| 16 | * subtract a constant offset to get to it. The pmd and pgd sizes are |
Helge Deller | 513e7ec | 2007-01-28 15:09:20 +0100 | [diff] [blame] | 17 | * arranged so that a single pmd covers 4GB (giving a full 64-bit |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | * process access to 8TB) so our lookups are effectively L2 for the |
| 19 | * first 4GB of the kernel (i.e. for all ILP32 processes and all the |
| 20 | * kernel for machines with under 4GB of memory) */ |
| 21 | static inline pgd_t *pgd_alloc(struct mm_struct *mm) |
| 22 | { |
| 23 | pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, |
| 24 | PGD_ALLOC_ORDER); |
| 25 | pgd_t *actual_pgd = pgd; |
| 26 | |
| 27 | if (likely(pgd != NULL)) { |
| 28 | memset(pgd, 0, PAGE_SIZE<<PGD_ALLOC_ORDER); |
Guenter Roeck | c19edb6 | 2015-04-15 13:34:28 -0700 | [diff] [blame] | 29 | #if CONFIG_PGTABLE_LEVELS == 3 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | actual_pgd += PTRS_PER_PGD; |
| 31 | /* Populate first pmd with allocated memory. We mark it |
| 32 | * with PxD_FLAG_ATTACHED as a signal to the system that this |
| 33 | * pmd entry may not be cleared. */ |
| 34 | __pgd_val_set(*actual_pgd, (PxD_FLAG_PRESENT | |
| 35 | PxD_FLAG_VALID | |
| 36 | PxD_FLAG_ATTACHED) |
| 37 | + (__u32)(__pa((unsigned long)pgd) >> PxD_VALUE_SHIFT)); |
Helge Deller | 2b3f344 | 2015-11-20 10:09:18 +0100 | [diff] [blame] | 38 | /* The first pmd entry also is marked with PxD_FLAG_ATTACHED as |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | * a signal that this pmd may not be freed */ |
| 40 | __pgd_val_set(*pgd, PxD_FLAG_ATTACHED); |
| 41 | #endif |
| 42 | } |
| 43 | return actual_pgd; |
| 44 | } |
| 45 | |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 46 | static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | { |
Guenter Roeck | c19edb6 | 2015-04-15 13:34:28 -0700 | [diff] [blame] | 48 | #if CONFIG_PGTABLE_LEVELS == 3 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | pgd -= PTRS_PER_PGD; |
| 50 | #endif |
| 51 | free_pages((unsigned long)pgd, PGD_ALLOC_ORDER); |
| 52 | } |
| 53 | |
Kirill A. Shutemov | f24ffde | 2015-04-14 15:45:54 -0700 | [diff] [blame] | 54 | #if CONFIG_PGTABLE_LEVELS == 3 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | |
| 56 | /* Three Level Page Table Support for pmd's */ |
| 57 | |
| 58 | static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) |
| 59 | { |
| 60 | __pgd_val_set(*pgd, (PxD_FLAG_PRESENT | PxD_FLAG_VALID) + |
| 61 | (__u32)(__pa((unsigned long)pmd) >> PxD_VALUE_SHIFT)); |
| 62 | } |
| 63 | |
| 64 | static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) |
| 65 | { |
| 66 | pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, |
| 67 | PMD_ORDER); |
| 68 | if (pmd) |
| 69 | memset(pmd, 0, PAGE_SIZE<<PMD_ORDER); |
| 70 | return pmd; |
| 71 | } |
| 72 | |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 73 | static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | { |
Christophe Jaillet | 4c4ac9a | 2015-07-13 11:32:43 +0200 | [diff] [blame] | 75 | if (pmd_flag(*pmd) & PxD_FLAG_ATTACHED) { |
Mikulas Patocka | 0e0da48 | 2015-03-18 13:42:38 -0400 | [diff] [blame] | 76 | /* |
| 77 | * This is the permanent pmd attached to the pgd; |
| 78 | * cannot free it. |
| 79 | * Increment the counter to compensate for the decrement |
| 80 | * done by generic mm code. |
| 81 | */ |
| 82 | mm_inc_nr_pmds(mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 83 | return; |
Christophe Jaillet | 4c4ac9a | 2015-07-13 11:32:43 +0200 | [diff] [blame] | 84 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 85 | free_pages((unsigned long)pmd, PMD_ORDER); |
| 86 | } |
| 87 | |
| 88 | #else |
| 89 | |
| 90 | /* Two Level Page Table Support for pmd's */ |
| 91 | |
| 92 | /* |
| 93 | * allocating and freeing a pmd is trivial: the 1-entry pmd is |
| 94 | * inside the pgd, so has no extra memory associated with it. |
| 95 | */ |
| 96 | |
| 97 | #define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); }) |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 98 | #define pmd_free(mm, x) do { } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | #define pgd_populate(mm, pmd, pte) BUG() |
| 100 | |
| 101 | #endif |
| 102 | |
| 103 | static inline void |
| 104 | pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) |
| 105 | { |
Guenter Roeck | c19edb6 | 2015-04-15 13:34:28 -0700 | [diff] [blame] | 106 | #if CONFIG_PGTABLE_LEVELS == 3 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 107 | /* preserve the gateway marker if this is the beginning of |
| 108 | * the permanent pmd */ |
| 109 | if(pmd_flag(*pmd) & PxD_FLAG_ATTACHED) |
| 110 | __pmd_val_set(*pmd, (PxD_FLAG_PRESENT | |
| 111 | PxD_FLAG_VALID | |
| 112 | PxD_FLAG_ATTACHED) |
| 113 | + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT)); |
| 114 | else |
| 115 | #endif |
| 116 | __pmd_val_set(*pmd, (PxD_FLAG_PRESENT | PxD_FLAG_VALID) |
| 117 | + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT)); |
| 118 | } |
| 119 | |
| 120 | #define pmd_populate(mm, pmd, pte_page) \ |
| 121 | pmd_populate_kernel(mm, pmd, page_address(pte_page)) |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 122 | #define pmd_pgtable(pmd) pmd_page(pmd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 124 | static inline pgtable_t |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 125 | pte_alloc_one(struct mm_struct *mm, unsigned long address) |
| 126 | { |
| 127 | struct page *page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); |
Kirill A. Shutemov | bc16640 | 2013-11-14 14:31:37 -0800 | [diff] [blame] | 128 | if (!page) |
| 129 | return NULL; |
| 130 | if (!pgtable_page_ctor(page)) { |
| 131 | __free_page(page); |
| 132 | return NULL; |
| 133 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | return page; |
| 135 | } |
| 136 | |
| 137 | static inline pte_t * |
| 138 | pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) |
| 139 | { |
| 140 | pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); |
| 141 | return pte; |
| 142 | } |
| 143 | |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 144 | static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 145 | { |
| 146 | free_page((unsigned long)pte); |
| 147 | } |
| 148 | |
Kyle McMartin | 9aa150b | 2008-02-18 14:16:26 -0800 | [diff] [blame] | 149 | static inline void pte_free(struct mm_struct *mm, struct page *pte) |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 150 | { |
| 151 | pgtable_page_dtor(pte); |
Kyle McMartin | 9aa150b | 2008-02-18 14:16:26 -0800 | [diff] [blame] | 152 | pte_free_kernel(mm, page_address(pte)); |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 153 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 154 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | #define check_pgt_cache() do { } while (0) |
| 156 | |
| 157 | #endif |