blob: ffae5dc8b3cef06c22b2ab65a0ec587089d3314c [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 * S390 version
Heiko Carstensa53c8fa2012-07-20 11:15:04 +02004 * Copyright IBM Corp. 1999, 2000
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * Author(s): Hartmut Penner (hp@de.ibm.com)
6 * Ulrich Weigand (weigand@de.ibm.com)
7 * Martin Schwidefsky (schwidefsky@de.ibm.com)
8 *
9 * Derived from "include/asm-i386/pgtable.h"
10 */
11
12#ifndef _ASM_S390_PGTABLE_H
13#define _ASM_S390_PGTABLE_H
14
Heiko Carstens9789db02008-07-14 09:59:11 +020015#include <linux/sched.h>
Heiko Carstens2dcea572006-09-29 01:58:41 -070016#include <linux/mm_types.h>
Martin Schwidefskyabf09be2012-11-07 13:17:37 +010017#include <linux/page-flags.h>
Martin Schwidefsky527e30b2014-04-30 16:04:25 +020018#include <linux/radix-tree.h>
Heiko Carstens37cd9442016-05-20 08:08:14 +020019#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <asm/bug.h>
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +020021#include <asm/page.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
Heiko Carstens0ccb32c2016-05-28 10:03:55 +020023extern pgd_t swapper_pg_dir[];
Linus Torvalds1da177e2005-04-16 15:20:36 -070024extern void paging_init(void);
25
Heiko Carstens37cd9442016-05-20 08:08:14 +020026enum {
27 PG_DIRECT_MAP_4K = 0,
28 PG_DIRECT_MAP_1M,
29 PG_DIRECT_MAP_2G,
30 PG_DIRECT_MAP_MAX
31};
32
33extern atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX];
34
35static inline void update_page_count(int level, long count)
36{
37 if (IS_ENABLED(CONFIG_PROC_FS))
38 atomic_long_add(count, &direct_pages_count[level]);
39}
40
41struct seq_file;
42void arch_report_meminfo(struct seq_file *m);
43
Linus Torvalds1da177e2005-04-16 15:20:36 -070044/*
45 * The S390 doesn't have any external MMU info: the kernel page
46 * tables contain all the necessary information.
47 */
Russell King4b3073e2009-12-18 16:40:18 +000048#define update_mmu_cache(vma, address, ptep) do { } while (0)
David Millerb113da62012-10-08 16:34:25 -070049#define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51/*
Martin Schwidefsky238ec4e2010-10-25 16:10:07 +020052 * ZERO_PAGE is a global shared page that is always zero; used
Linus Torvalds1da177e2005-04-16 15:20:36 -070053 * for zero-mapped memory areas etc..
54 */
Martin Schwidefsky238ec4e2010-10-25 16:10:07 +020055
56extern unsigned long empty_zero_page;
57extern unsigned long zero_page_mask;
58
59#define ZERO_PAGE(vaddr) \
60 (virt_to_page((void *)(empty_zero_page + \
61 (((unsigned long)(vaddr)) &zero_page_mask))))
Kirill A. Shutemov816422a2012-12-12 13:52:36 -080062#define __HAVE_COLOR_ZERO_PAGE
Martin Schwidefsky238ec4e2010-10-25 16:10:07 +020063
Linus Torvalds4f2e2902013-04-17 08:46:19 -070064/* TODO: s390 cannot support io_remap_pfn_range... */
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
Kirill A. Shutemovd016bf72015-02-11 15:26:41 -080066#define FIRST_USER_ADDRESS 0UL
Hugh Dickinsd455a362005-04-19 13:29:23 -070067
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#define pte_ERROR(e) \
69 printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e))
70#define pmd_ERROR(e) \
71 printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e))
Martin Schwidefsky190a1d72007-10-22 12:52:48 +020072#define pud_ERROR(e) \
73 printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e))
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +020074#define p4d_ERROR(e) \
75 printk("%s:%d: bad p4d %p.\n", __FILE__, __LINE__, (void *) p4d_val(e))
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#define pgd_ERROR(e) \
77 printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e))
78
Linus Torvalds1da177e2005-04-16 15:20:36 -070079/*
Martin Schwidefskya1c843b2015-04-22 13:55:59 +020080 * The vmalloc and module area will always be on the topmost area of the
81 * kernel mapping. We reserve 128GB (64bit) for vmalloc and modules.
Heiko Carstensc972cc62012-10-05 16:52:18 +020082 * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where
83 * modules will reside. That makes sure that inter module branches always
84 * happen without trampolines and in addition the placement within a 2GB frame
85 * is branch prediction unit friendly.
Heiko Carstens8b62bc92006-12-04 15:40:56 +010086 */
Heiko Carstens239a64252009-06-12 10:26:33 +020087extern unsigned long VMALLOC_START;
Martin Schwidefsky14045eb2011-12-27 11:27:07 +010088extern unsigned long VMALLOC_END;
89extern struct page *vmemmap;
Heiko Carstens239a64252009-06-12 10:26:33 +020090
Martin Schwidefsky14045eb2011-12-27 11:27:07 +010091#define VMEM_MAX_PHYS ((unsigned long) vmemmap)
Christian Borntraeger5fd9c6e2008-01-26 14:11:00 +010092
Heiko Carstensc972cc62012-10-05 16:52:18 +020093extern unsigned long MODULES_VADDR;
94extern unsigned long MODULES_END;
95#define MODULES_VADDR MODULES_VADDR
96#define MODULES_END MODULES_END
97#define MODULES_LEN (1UL << 31)
Heiko Carstensc972cc62012-10-05 16:52:18 +020098
Heiko Carstensc9331462014-10-15 12:17:38 +020099static inline int is_module_addr(void *addr)
100{
Heiko Carstensc9331462014-10-15 12:17:38 +0200101 BUILD_BUG_ON(MODULES_LEN > (1UL << 31));
102 if (addr < (void *)MODULES_VADDR)
103 return 0;
104 if (addr > (void *)MODULES_END)
105 return 0;
Heiko Carstensc9331462014-10-15 12:17:38 +0200106 return 1;
107}
108
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 * A 64 bit pagetable entry of S390 has following format:
Christian Borntraeger6a985c62009-12-07 12:52:11 +0100111 * | PFRA |0IPC| OS |
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 * 0000000000111111111122222222223333333333444444444455555555556666
113 * 0123456789012345678901234567890123456789012345678901234567890123
114 *
115 * I Page-Invalid Bit: Page is not available for address-translation
116 * P Page-Protection Bit: Store access not possible for page
Christian Borntraeger6a985c62009-12-07 12:52:11 +0100117 * C Change-bit override: HW is not required to set change bit
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 *
119 * A 64 bit segmenttable entry of S390 has following format:
120 * | P-table origin | TT
121 * 0000000000111111111122222222223333333333444444444455555555556666
122 * 0123456789012345678901234567890123456789012345678901234567890123
123 *
124 * I Segment-Invalid Bit: Segment is not available for address-translation
125 * C Common-Segment Bit: Segment is not private (PoP 3-30)
126 * P Page-Protection Bit: Store access not possible for page
127 * TT Type 00
128 *
129 * A 64 bit region table entry of S390 has following format:
130 * | S-table origin | TF TTTL
131 * 0000000000111111111122222222223333333333444444444455555555556666
132 * 0123456789012345678901234567890123456789012345678901234567890123
133 *
134 * I Segment-Invalid Bit: Segment is not available for address-translation
135 * TT Type 01
136 * TF
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200137 * TL Table length
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 *
139 * The 64 bit regiontable origin of S390 has following format:
140 * | region table origon | DTTL
141 * 0000000000111111111122222222223333333333444444444455555555556666
142 * 0123456789012345678901234567890123456789012345678901234567890123
143 *
144 * X Space-Switch event:
145 * G Segment-Invalid Bit:
146 * P Private-Space Bit:
147 * S Storage-Alteration:
148 * R Real space
149 * TL Table-Length:
150 *
151 * A storage key has the following format:
152 * | ACC |F|R|C|0|
153 * 0 3 4 5 6 7
154 * ACC: access key
155 * F : fetch protection bit
156 * R : referenced bit
157 * C : changed bit
158 */
159
160/* Hardware bits in the page table entry */
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100161#define _PAGE_NOEXEC 0x100 /* HW no-execute bit */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200162#define _PAGE_PROTECT 0x200 /* HW read-only bit */
Martin Schwidefsky83377482006-10-18 18:30:51 +0200163#define _PAGE_INVALID 0x400 /* HW invalid bit */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200164#define _PAGE_LARGE 0x800 /* Bit to mark a large pte */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200165
166/* Software bits in the page table entry */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200167#define _PAGE_PRESENT 0x001 /* SW pte present bit */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200168#define _PAGE_YOUNG 0x004 /* SW pte young bit */
169#define _PAGE_DIRTY 0x008 /* SW pte dirty bit */
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200170#define _PAGE_READ 0x010 /* SW pte read bit */
171#define _PAGE_WRITE 0x020 /* SW pte write bit */
172#define _PAGE_SPECIAL 0x040 /* SW associated with special page */
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200173#define _PAGE_UNUSED 0x080 /* SW bit for pgste usage state */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
Martin Schwidefsky5614dd92015-04-22 14:47:42 +0200175#ifdef CONFIG_MEM_SOFT_DIRTY
176#define _PAGE_SOFT_DIRTY 0x002 /* SW pte soft dirty bit */
177#else
178#define _PAGE_SOFT_DIRTY 0x000
179#endif
180
Nick Piggin138c9022008-07-08 11:31:06 +0200181/* Set of bits not changed in pte_modify */
Heiko Carstens6a5c1482014-09-22 08:50:51 +0200182#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_DIRTY | \
Martin Schwidefsky5614dd92015-04-22 14:47:42 +0200183 _PAGE_YOUNG | _PAGE_SOFT_DIRTY)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
Martin Schwidefsky83377482006-10-18 18:30:51 +0200185/*
Kirill A. Shutemov6e76d4b2015-02-10 14:11:04 -0800186 * handle_pte_fault uses pte_present and pte_none to find out the pte type
187 * WITHOUT holding the page table lock. The _PAGE_PRESENT bit is used to
188 * distinguish present from not-present ptes. It is changed only with the page
189 * table lock held.
Martin Schwidefsky83377482006-10-18 18:30:51 +0200190 *
Martin Schwidefskye5098612013-07-23 20:57:57 +0200191 * The following table gives the different possible bit combinations for
Martin Schwidefskya1c843b2015-04-22 13:55:59 +0200192 * the pte hardware and software bits in the last 12 bits of a pte
193 * (. unassigned bit, x don't care, t swap type):
Martin Schwidefsky83377482006-10-18 18:30:51 +0200194 *
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200195 * 842100000000
196 * 000084210000
197 * 000000008421
Martin Schwidefskya1c843b2015-04-22 13:55:59 +0200198 * .IR.uswrdy.p
199 * empty .10.00000000
200 * swap .11..ttttt.0
201 * prot-none, clean, old .11.xx0000.1
202 * prot-none, clean, young .11.xx0001.1
Gerald Schaeferbc29b7a2016-07-18 14:35:13 +0200203 * prot-none, dirty, old .11.xx0010.1
204 * prot-none, dirty, young .11.xx0011.1
Martin Schwidefskya1c843b2015-04-22 13:55:59 +0200205 * read-only, clean, old .11.xx0100.1
206 * read-only, clean, young .01.xx0101.1
207 * read-only, dirty, old .11.xx0110.1
208 * read-only, dirty, young .01.xx0111.1
209 * read-write, clean, old .11.xx1100.1
210 * read-write, clean, young .01.xx1101.1
211 * read-write, dirty, old .10.xx1110.1
212 * read-write, dirty, young .00.xx1111.1
213 * HW-bits: R read-only, I invalid
214 * SW-bits: p present, y young, d dirty, r read, w write, s special,
215 * u unused, l large
Martin Schwidefskye5098612013-07-23 20:57:57 +0200216 *
Martin Schwidefskya1c843b2015-04-22 13:55:59 +0200217 * pte_none is true for the bit pattern .10.00000000, pte == 0x400
218 * pte_swap is true for the bit pattern .11..ooooo.0, (pte & 0x201) == 0x200
219 * pte_present is true for the bit pattern .xx.xxxxxx.1, (pte & 0x001) == 0x001
Martin Schwidefsky83377482006-10-18 18:30:51 +0200220 */
221
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200222/* Bits in the segment/region table address-space-control-element */
Heiko Carstens8457d772017-06-14 08:57:24 +0200223#define _ASCE_ORIGIN ~0xfffUL/* region/segment table origin */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200224#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */
225#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */
226#define _ASCE_SPACE_SWITCH 0x40 /* space switch event */
227#define _ASCE_REAL_SPACE 0x20 /* real space control */
228#define _ASCE_TYPE_MASK 0x0c /* asce table type mask */
229#define _ASCE_TYPE_REGION1 0x0c /* region first table type */
230#define _ASCE_TYPE_REGION2 0x08 /* region second table type */
231#define _ASCE_TYPE_REGION3 0x04 /* region third table type */
232#define _ASCE_TYPE_SEGMENT 0x00 /* segment table type */
233#define _ASCE_TABLE_LENGTH 0x03 /* region table length */
234
235/* Bits in the region table entry */
236#define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200237#define _REGION_ENTRY_PROTECT 0x200 /* region protection bit */
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100238#define _REGION_ENTRY_NOEXEC 0x100 /* region no-execute bit */
Martin Schwidefsky4be130a2016-03-08 12:12:18 +0100239#define _REGION_ENTRY_OFFSET 0xc0 /* region table offset */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200240#define _REGION_ENTRY_INVALID 0x20 /* invalid region table entry */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200241#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */
242#define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */
243#define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */
244#define _REGION_ENTRY_TYPE_R3 0x04 /* region third table type */
245#define _REGION_ENTRY_LENGTH 0x03 /* region third length */
246
247#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
Martin Schwidefskye5098612013-07-23 20:57:57 +0200248#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID)
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200249#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
Martin Schwidefskye5098612013-07-23 20:57:57 +0200250#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID)
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200251#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
Martin Schwidefskye5098612013-07-23 20:57:57 +0200252#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID)
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200253
Heiko Carstens9e20b4d2016-05-10 10:34:47 +0200254#define _REGION3_ENTRY_ORIGIN_LARGE ~0x7fffffffUL /* large page address */
Heiko Carstens2dffdcb2016-05-11 10:52:07 +0200255#define _REGION3_ENTRY_DIRTY 0x2000 /* SW region dirty bit */
256#define _REGION3_ENTRY_YOUNG 0x1000 /* SW region young bit */
257#define _REGION3_ENTRY_LARGE 0x0400 /* RTTE-format control, large page */
258#define _REGION3_ENTRY_READ 0x0002 /* SW region read bit */
259#define _REGION3_ENTRY_WRITE 0x0001 /* SW region write bit */
260
261#ifdef CONFIG_MEM_SOFT_DIRTY
262#define _REGION3_ENTRY_SOFT_DIRTY 0x4000 /* SW region soft dirty bit */
263#else
264#define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */
265#endif
266
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +0200267#define _REGION_ENTRY_BITS 0xfffffffffffff22fUL
268#define _REGION_ENTRY_BITS_LARGE 0xffffffff8000fe2fUL
Gerald Schaeferd08de8e2016-07-04 14:47:01 +0200269
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200270/* Bits in the segment table entry */
Janosch Frank58b7e202018-07-13 11:28:20 +0100271#define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL
272#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL
273#define _SEGMENT_ENTRY_HARDWARE_BITS 0xfffffffffffffe30UL
274#define _SEGMENT_ENTRY_HARDWARE_BITS_LARGE 0xfffffffffff00730UL
Heiko Carstensea815312013-03-21 12:50:39 +0100275#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */
Heiko Carstens8457d772017-06-14 08:57:24 +0200276#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* page table origin */
277#define _SEGMENT_ENTRY_PROTECT 0x200 /* segment protection bit */
278#define _SEGMENT_ENTRY_NOEXEC 0x100 /* segment no-execute bit */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200279#define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200280
281#define _SEGMENT_ENTRY (0)
Martin Schwidefskye5098612013-07-23 20:57:57 +0200282#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID)
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200283
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200284#define _SEGMENT_ENTRY_DIRTY 0x2000 /* SW segment dirty bit */
285#define _SEGMENT_ENTRY_YOUNG 0x1000 /* SW segment young bit */
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200286#define _SEGMENT_ENTRY_LARGE 0x0400 /* STE-format control, large page */
Gerald Schaeferbc29b7a2016-07-18 14:35:13 +0200287#define _SEGMENT_ENTRY_WRITE 0x0002 /* SW segment write bit */
288#define _SEGMENT_ENTRY_READ 0x0001 /* SW segment read bit */
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200289
Martin Schwidefsky5614dd92015-04-22 14:47:42 +0200290#ifdef CONFIG_MEM_SOFT_DIRTY
291#define _SEGMENT_ENTRY_SOFT_DIRTY 0x4000 /* SW segment soft dirty bit */
292#else
293#define _SEGMENT_ENTRY_SOFT_DIRTY 0x0000 /* SW segment soft dirty bit */
294#endif
295
Heiko Carstensc67da7c2017-06-16 17:24:39 +0200296#define _CRST_ENTRIES 2048 /* number of region/segment table entries */
297#define _PAGE_ENTRIES 256 /* number of page table entries */
298
299#define _CRST_TABLE_SIZE (_CRST_ENTRIES * 8)
300#define _PAGE_TABLE_SIZE (_PAGE_ENTRIES * 8)
301
302#define _REGION1_SHIFT 53
303#define _REGION2_SHIFT 42
304#define _REGION3_SHIFT 31
305#define _SEGMENT_SHIFT 20
306
307#define _REGION1_INDEX (0x7ffUL << _REGION1_SHIFT)
308#define _REGION2_INDEX (0x7ffUL << _REGION2_SHIFT)
309#define _REGION3_INDEX (0x7ffUL << _REGION3_SHIFT)
310#define _SEGMENT_INDEX (0x7ffUL << _SEGMENT_SHIFT)
311#define _PAGE_INDEX (0xffUL << _PAGE_SHIFT)
312
313#define _REGION1_SIZE (1UL << _REGION1_SHIFT)
314#define _REGION2_SIZE (1UL << _REGION2_SHIFT)
315#define _REGION3_SIZE (1UL << _REGION3_SHIFT)
316#define _SEGMENT_SIZE (1UL << _SEGMENT_SHIFT)
317
318#define _REGION1_MASK (~(_REGION1_SIZE - 1))
319#define _REGION2_MASK (~(_REGION2_SIZE - 1))
320#define _REGION3_MASK (~(_REGION3_SIZE - 1))
321#define _SEGMENT_MASK (~(_SEGMENT_SIZE - 1))
322
323#define PMD_SHIFT _SEGMENT_SHIFT
324#define PUD_SHIFT _REGION3_SHIFT
325#define P4D_SHIFT _REGION2_SHIFT
326#define PGDIR_SHIFT _REGION1_SHIFT
327
328#define PMD_SIZE _SEGMENT_SIZE
329#define PUD_SIZE _REGION3_SIZE
330#define P4D_SIZE _REGION2_SIZE
331#define PGDIR_SIZE _REGION1_SIZE
332
333#define PMD_MASK _SEGMENT_MASK
334#define PUD_MASK _REGION3_MASK
335#define P4D_MASK _REGION2_MASK
336#define PGDIR_MASK _REGION1_MASK
337
338#define PTRS_PER_PTE _PAGE_ENTRIES
339#define PTRS_PER_PMD _CRST_ENTRIES
340#define PTRS_PER_PUD _CRST_ENTRIES
341#define PTRS_PER_P4D _CRST_ENTRIES
342#define PTRS_PER_PGD _CRST_ENTRIES
343
Vasily Gorbik34377d32018-09-12 13:23:58 +0200344#define MAX_PTRS_PER_P4D PTRS_PER_P4D
345
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200346/*
Heiko Carstens2dffdcb2016-05-11 10:52:07 +0200347 * Segment table and region3 table entry encoding
348 * (R = read-only, I = invalid, y = young bit):
Gerald Schaeferbc29b7a2016-07-18 14:35:13 +0200349 * dy..R...I...wr
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200350 * prot-none, clean, old 00..1...1...00
351 * prot-none, clean, young 01..1...1...00
352 * prot-none, dirty, old 10..1...1...00
353 * prot-none, dirty, young 11..1...1...00
Gerald Schaeferbc29b7a2016-07-18 14:35:13 +0200354 * read-only, clean, old 00..1...1...01
355 * read-only, clean, young 01..1...0...01
356 * read-only, dirty, old 10..1...1...01
357 * read-only, dirty, young 11..1...0...01
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200358 * read-write, clean, old 00..1...1...11
359 * read-write, clean, young 01..1...0...11
360 * read-write, dirty, old 10..0...1...11
361 * read-write, dirty, young 11..0...0...11
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200362 * The segment table origin is used to distinguish empty (origin==0) from
363 * read-write, old segment table entries (origin!=0)
Martin Schwidefskya1c843b2015-04-22 13:55:59 +0200364 * HW-bits: R read-only, I invalid
365 * SW-bits: y young, d dirty, r read, w write
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200366 */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200367
Martin Schwidefsky6c61cfe2011-06-06 14:14:42 +0200368/* Page status table bits for virtualization */
Martin Schwidefsky0d0dafc2013-05-17 14:41:33 +0200369#define PGSTE_ACC_BITS 0xf000000000000000UL
370#define PGSTE_FP_BIT 0x0800000000000000UL
371#define PGSTE_PCL_BIT 0x0080000000000000UL
372#define PGSTE_HR_BIT 0x0040000000000000UL
373#define PGSTE_HC_BIT 0x0020000000000000UL
374#define PGSTE_GR_BIT 0x0004000000000000UL
375#define PGSTE_GC_BIT 0x0002000000000000UL
Martin Schwidefsky0a61b222013-10-18 12:03:41 +0200376#define PGSTE_UC_BIT 0x0000800000000000UL /* user dirty (migration) */
377#define PGSTE_IN_BIT 0x0000400000000000UL /* IPTE notify bit */
Martin Schwidefsky4be130a2016-03-08 12:12:18 +0100378#define PGSTE_VSIE_BIT 0x0000200000000000UL /* ref'd in a shadow table */
Martin Schwidefsky6c61cfe2011-06-06 14:14:42 +0200379
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200380/* Guest Page State used for virtualization */
Claudio Imbrenda2d42f942017-04-20 10:03:45 +0200381#define _PGSTE_GPS_ZERO 0x0000000080000000UL
Martin Schwidefskycd774b92016-07-26 17:02:31 +0200382#define _PGSTE_GPS_NODAT 0x0000000040000000UL
Claudio Imbrenda2d42f942017-04-20 10:03:45 +0200383#define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL
384#define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL
385#define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL
386#define _PGSTE_GPS_USAGE_POT_VOLATILE 0x0000000002000000UL
387#define _PGSTE_GPS_USAGE_VOLATILE _PGSTE_GPS_USAGE_MASK
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200388
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200389/*
390 * A user page table pointer has the space-switch-event bit, the
391 * private-space-control bit and the storage-alteration-event-control
392 * bit set. A kernel page table pointer doesn't need them.
393 */
394#define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
395 _ASCE_ALT_EVENT)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397/*
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200398 * Page protection definitions.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 */
Gerald Schaeferbc29b7a2016-07-18 14:35:13 +0200400#define PAGE_NONE __pgprot(_PAGE_PRESENT | _PAGE_INVALID | _PAGE_PROTECT)
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100401#define PAGE_RO __pgprot(_PAGE_PRESENT | _PAGE_READ | \
402 _PAGE_NOEXEC | _PAGE_INVALID | _PAGE_PROTECT)
403#define PAGE_RX __pgprot(_PAGE_PRESENT | _PAGE_READ | \
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200404 _PAGE_INVALID | _PAGE_PROTECT)
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100405#define PAGE_RW __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
406 _PAGE_NOEXEC | _PAGE_INVALID | _PAGE_PROTECT)
407#define PAGE_RWX __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200408 _PAGE_INVALID | _PAGE_PROTECT)
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200409
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200410#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100411 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200412#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100413 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200414#define PAGE_KERNEL_RO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100415 _PAGE_PROTECT | _PAGE_NOEXEC)
416#define PAGE_KERNEL_EXEC __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
417 _PAGE_YOUNG | _PAGE_DIRTY)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418
419/*
Martin Schwidefsky043d0702011-05-23 10:24:23 +0200420 * On s390 the page table entry has an invalid bit and a read-only bit.
421 * Read permission implies execute permission and write permission
422 * implies read permission.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 */
424 /*xwr*/
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200425#define __P000 PAGE_NONE
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100426#define __P001 PAGE_RO
427#define __P010 PAGE_RO
428#define __P011 PAGE_RO
429#define __P100 PAGE_RX
430#define __P101 PAGE_RX
431#define __P110 PAGE_RX
432#define __P111 PAGE_RX
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200434#define __S000 PAGE_NONE
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100435#define __S001 PAGE_RO
436#define __S010 PAGE_RW
437#define __S011 PAGE_RW
438#define __S100 PAGE_RX
439#define __S101 PAGE_RX
440#define __S110 PAGE_RWX
441#define __S111 PAGE_RWX
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442
Gerald Schaefer106c9922013-04-29 15:07:23 -0700443/*
444 * Segment entry (large page) protection definitions.
445 */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200446#define SEGMENT_NONE __pgprot(_SEGMENT_ENTRY_INVALID | \
447 _SEGMENT_ENTRY_PROTECT)
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100448#define SEGMENT_RO __pgprot(_SEGMENT_ENTRY_PROTECT | \
449 _SEGMENT_ENTRY_READ | \
450 _SEGMENT_ENTRY_NOEXEC)
451#define SEGMENT_RX __pgprot(_SEGMENT_ENTRY_PROTECT | \
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200452 _SEGMENT_ENTRY_READ)
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100453#define SEGMENT_RW __pgprot(_SEGMENT_ENTRY_READ | \
454 _SEGMENT_ENTRY_WRITE | \
455 _SEGMENT_ENTRY_NOEXEC)
456#define SEGMENT_RWX __pgprot(_SEGMENT_ENTRY_READ | \
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200457 _SEGMENT_ENTRY_WRITE)
Heiko Carstens2dffdcb2016-05-11 10:52:07 +0200458#define SEGMENT_KERNEL __pgprot(_SEGMENT_ENTRY | \
459 _SEGMENT_ENTRY_LARGE | \
460 _SEGMENT_ENTRY_READ | \
461 _SEGMENT_ENTRY_WRITE | \
462 _SEGMENT_ENTRY_YOUNG | \
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100463 _SEGMENT_ENTRY_DIRTY | \
464 _SEGMENT_ENTRY_NOEXEC)
Heiko Carstens2dffdcb2016-05-11 10:52:07 +0200465#define SEGMENT_KERNEL_RO __pgprot(_SEGMENT_ENTRY | \
466 _SEGMENT_ENTRY_LARGE | \
467 _SEGMENT_ENTRY_READ | \
468 _SEGMENT_ENTRY_YOUNG | \
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100469 _SEGMENT_ENTRY_PROTECT | \
470 _SEGMENT_ENTRY_NOEXEC)
Heiko Carstens2dffdcb2016-05-11 10:52:07 +0200471
472/*
473 * Region3 entry (large page) protection definitions.
474 */
475
476#define REGION3_KERNEL __pgprot(_REGION_ENTRY_TYPE_R3 | \
477 _REGION3_ENTRY_LARGE | \
478 _REGION3_ENTRY_READ | \
479 _REGION3_ENTRY_WRITE | \
480 _REGION3_ENTRY_YOUNG | \
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100481 _REGION3_ENTRY_DIRTY | \
482 _REGION_ENTRY_NOEXEC)
Heiko Carstens2dffdcb2016-05-11 10:52:07 +0200483#define REGION3_KERNEL_RO __pgprot(_REGION_ENTRY_TYPE_R3 | \
484 _REGION3_ENTRY_LARGE | \
485 _REGION3_ENTRY_READ | \
486 _REGION3_ENTRY_YOUNG | \
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100487 _REGION_ENTRY_PROTECT | \
488 _REGION_ENTRY_NOEXEC)
Gerald Schaefer106c9922013-04-29 15:07:23 -0700489
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200490static inline int mm_has_pgste(struct mm_struct *mm)
491{
492#ifdef CONFIG_PGSTE
493 if (unlikely(mm->context.has_pgste))
494 return 1;
495#endif
496 return 0;
497}
Dominik Dingel65eef3352014-01-14 15:02:11 +0100498
Martin Schwidefsky0b46e0a2015-04-15 13:23:26 +0200499static inline int mm_alloc_pgste(struct mm_struct *mm)
500{
501#ifdef CONFIG_PGSTE
502 if (unlikely(mm->context.alloc_pgste))
503 return 1;
504#endif
505 return 0;
506}
507
Dominik Dingel2faee8f2014-10-23 12:08:38 +0200508/*
509 * In the case that a guest uses storage keys
510 * faults should no longer be backed by zero pages
511 */
Christian Borntraegerfa41ba02017-08-24 12:55:08 +0200512#define mm_forbids_zeropage mm_has_pgste
Janosch Frank55531b72018-02-15 16:33:47 +0100513static inline int mm_uses_skeys(struct mm_struct *mm)
Dominik Dingel65eef3352014-01-14 15:02:11 +0100514{
515#ifdef CONFIG_PGSTE
Janosch Frank55531b72018-02-15 16:33:47 +0100516 if (mm->context.uses_skeys)
Dominik Dingel65eef3352014-01-14 15:02:11 +0100517 return 1;
518#endif
519 return 0;
520}
521
Heiko Carstens4ccccc52016-05-14 10:46:33 +0200522static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new)
523{
524 register unsigned long reg2 asm("2") = old;
525 register unsigned long reg3 asm("3") = new;
526 unsigned long address = (unsigned long)ptr | 1;
527
528 asm volatile(
529 " csp %0,%3"
530 : "+d" (reg2), "+m" (*ptr)
531 : "d" (reg3), "d" (address)
532 : "cc");
533}
534
Heiko Carstense8a97e42016-05-17 10:50:15 +0200535static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new)
536{
537 register unsigned long reg2 asm("2") = old;
538 register unsigned long reg3 asm("3") = new;
539 unsigned long address = (unsigned long)ptr | 1;
540
541 asm volatile(
542 " .insn rre,0xb98a0000,%0,%3"
543 : "+d" (reg2), "+m" (*ptr)
544 : "d" (reg3), "d" (address)
545 : "cc");
546}
547
548#define CRDTE_DTT_PAGE 0x00UL
549#define CRDTE_DTT_SEGMENT 0x10UL
550#define CRDTE_DTT_REGION3 0x14UL
551#define CRDTE_DTT_REGION2 0x18UL
552#define CRDTE_DTT_REGION1 0x1cUL
553
554static inline void crdte(unsigned long old, unsigned long new,
555 unsigned long table, unsigned long dtt,
556 unsigned long address, unsigned long asce)
557{
558 register unsigned long reg2 asm("2") = old;
559 register unsigned long reg3 asm("3") = new;
560 register unsigned long reg4 asm("4") = table | dtt;
561 register unsigned long reg5 asm("5") = address;
562
563 asm volatile(".insn rrf,0xb98f0000,%0,%2,%4,0"
564 : "+d" (reg2)
565 : "d" (reg3), "d" (reg4), "d" (reg5), "a" (asce)
566 : "memory", "cc");
567}
568
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569/*
Heiko Carstenscc18b462017-05-20 11:43:26 +0200570 * pgd/p4d/pud/pmd/pte query functions
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 */
Heiko Carstenscc18b462017-05-20 11:43:26 +0200572static inline int pgd_folded(pgd_t pgd)
573{
574 return (pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1;
575}
576
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100577static inline int pgd_present(pgd_t pgd)
578{
Heiko Carstenscc18b462017-05-20 11:43:26 +0200579 if (pgd_folded(pgd))
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100580 return 1;
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100581 return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
582}
583
584static inline int pgd_none(pgd_t pgd)
585{
Heiko Carstenscc18b462017-05-20 11:43:26 +0200586 if (pgd_folded(pgd))
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100587 return 0;
Martin Schwidefskye5098612013-07-23 20:57:57 +0200588 return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL;
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100589}
590
591static inline int pgd_bad(pgd_t pgd)
592{
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100593 /*
594 * With dynamic page table levels the pgd can be a region table
595 * entry or a segment table entry. Check for the bit that are
596 * invalid for either table entry.
597 */
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100598 unsigned long mask =
Martin Schwidefskye5098612013-07-23 20:57:57 +0200599 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID &
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100600 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
601 return (pgd_val(pgd) & mask) != 0;
602}
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200603
Heiko Carstenscc18b462017-05-20 11:43:26 +0200604static inline int p4d_folded(p4d_t p4d)
605{
606 return (p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2;
607}
608
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +0200609static inline int p4d_present(p4d_t p4d)
610{
Heiko Carstenscc18b462017-05-20 11:43:26 +0200611 if (p4d_folded(p4d))
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +0200612 return 1;
613 return (p4d_val(p4d) & _REGION_ENTRY_ORIGIN) != 0UL;
614}
615
616static inline int p4d_none(p4d_t p4d)
617{
Heiko Carstenscc18b462017-05-20 11:43:26 +0200618 if (p4d_folded(p4d))
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +0200619 return 0;
620 return p4d_val(p4d) == _REGION2_ENTRY_EMPTY;
621}
622
623static inline unsigned long p4d_pfn(p4d_t p4d)
624{
625 unsigned long origin_mask;
626
627 origin_mask = _REGION_ENTRY_ORIGIN;
628 return (p4d_val(p4d) & origin_mask) >> PAGE_SHIFT;
629}
630
Heiko Carstenscc18b462017-05-20 11:43:26 +0200631static inline int pud_folded(pud_t pud)
632{
633 return (pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3;
634}
635
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200636static inline int pud_present(pud_t pud)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637{
Heiko Carstenscc18b462017-05-20 11:43:26 +0200638 if (pud_folded(pud))
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100639 return 1;
Martin Schwidefsky0d017922007-12-17 16:25:48 +0100640 return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641}
642
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200643static inline int pud_none(pud_t pud)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644{
Heiko Carstenscc18b462017-05-20 11:43:26 +0200645 if (pud_folded(pud))
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100646 return 0;
Gerald Schaeferd08de8e2016-07-04 14:47:01 +0200647 return pud_val(pud) == _REGION3_ENTRY_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648}
649
Heiko Carstens18da2362012-10-08 09:18:26 +0200650static inline int pud_large(pud_t pud)
651{
652 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
653 return 0;
654 return !!(pud_val(pud) & _REGION3_ENTRY_LARGE);
655}
656
Heiko Carstens9e20b4d2016-05-10 10:34:47 +0200657static inline unsigned long pud_pfn(pud_t pud)
658{
659 unsigned long origin_mask;
660
Heiko Carstensf96c6f72017-05-22 13:27:34 +0200661 origin_mask = _REGION_ENTRY_ORIGIN;
Heiko Carstens9e20b4d2016-05-10 10:34:47 +0200662 if (pud_large(pud))
663 origin_mask = _REGION3_ENTRY_ORIGIN_LARGE;
664 return (pud_val(pud) & origin_mask) >> PAGE_SHIFT;
665}
666
Gerald Schaeferd08de8e2016-07-04 14:47:01 +0200667static inline int pmd_large(pmd_t pmd)
668{
669 return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0;
670}
671
672static inline int pmd_bad(pmd_t pmd)
673{
674 if (pmd_large(pmd))
675 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0;
676 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
677}
678
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200679static inline int pud_bad(pud_t pud)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680{
Gerald Schaeferd08de8e2016-07-04 14:47:01 +0200681 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
682 return pmd_bad(__pmd(pud_val(pud)));
683 if (pud_large(pud))
684 return (pud_val(pud) & ~_REGION_ENTRY_BITS_LARGE) != 0;
685 return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686}
687
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +0200688static inline int p4d_bad(p4d_t p4d)
689{
690 if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
691 return pud_bad(__pud(p4d_val(p4d)));
692 return (p4d_val(p4d) & ~_REGION_ENTRY_BITS) != 0;
693}
694
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800695static inline int pmd_present(pmd_t pmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696{
Dominik Dingel54397bb2016-04-27 11:43:07 +0200697 return pmd_val(pmd) != _SEGMENT_ENTRY_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698}
699
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800700static inline int pmd_none(pmd_t pmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701{
Dominik Dingel54397bb2016-04-27 11:43:07 +0200702 return pmd_val(pmd) == _SEGMENT_ENTRY_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703}
704
Martin Schwidefsky7cded342015-05-13 14:33:22 +0200705static inline unsigned long pmd_pfn(pmd_t pmd)
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200706{
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200707 unsigned long origin_mask;
708
709 origin_mask = _SEGMENT_ENTRY_ORIGIN;
710 if (pmd_large(pmd))
711 origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE;
712 return (pmd_val(pmd) & origin_mask) >> PAGE_SHIFT;
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200713}
714
Dan Williamse4e40e02017-11-29 16:10:10 -0800715#define pmd_write pmd_write
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -0700716static inline int pmd_write(pmd_t pmd)
717{
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200718 return (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) != 0;
719}
720
721static inline int pmd_dirty(pmd_t pmd)
722{
723 int dirty = 1;
724 if (pmd_large(pmd))
725 dirty = (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) != 0;
726 return dirty;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -0700727}
728
729static inline int pmd_young(pmd_t pmd)
730{
Martin Schwidefsky152125b2014-07-24 11:03:41 +0200731 int young = 1;
732 if (pmd_large(pmd))
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200733 young = (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0;
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200734 return young;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -0700735}
736
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800737static inline int pte_present(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200739 /* Bit pattern: (pte & 0x001) == 0x001 */
740 return (pte_val(pte) & _PAGE_PRESENT) != 0;
741}
742
743static inline int pte_none(pte_t pte)
744{
745 /* Bit pattern: pte == 0x400 */
746 return pte_val(pte) == _PAGE_INVALID;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747}
748
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200749static inline int pte_swap(pte_t pte)
750{
Martin Schwidefskya1c843b2015-04-22 13:55:59 +0200751 /* Bit pattern: (pte & 0x201) == 0x200 */
752 return (pte_val(pte) & (_PAGE_PROTECT | _PAGE_PRESENT))
753 == _PAGE_PROTECT;
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200754}
755
Nick Piggin7e675132008-04-28 02:13:00 -0700756static inline int pte_special(pte_t pte)
757{
Nick Piggina08cb622008-04-28 02:13:03 -0700758 return (pte_val(pte) & _PAGE_SPECIAL);
Nick Piggin7e675132008-04-28 02:13:00 -0700759}
760
Martin Schwidefskyba8a9222007-10-22 12:52:44 +0200761#define __HAVE_ARCH_PTE_SAME
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200762static inline int pte_same(pte_t a, pte_t b)
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100763{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200764 return pte_val(a) == pte_val(b);
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100765}
766
Martin Schwidefskyb54565b2014-09-23 14:01:34 +0200767#ifdef CONFIG_NUMA_BALANCING
768static inline int pte_protnone(pte_t pte)
769{
770 return pte_present(pte) && !(pte_val(pte) & _PAGE_READ);
771}
772
773static inline int pmd_protnone(pmd_t pmd)
774{
775 /* pmd_large(pmd) implies pmd_present(pmd) */
776 return pmd_large(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ);
777}
778#endif
779
Martin Schwidefsky5614dd92015-04-22 14:47:42 +0200780static inline int pte_soft_dirty(pte_t pte)
781{
782 return pte_val(pte) & _PAGE_SOFT_DIRTY;
783}
784#define pte_swp_soft_dirty pte_soft_dirty
785
786static inline pte_t pte_mksoft_dirty(pte_t pte)
787{
788 pte_val(pte) |= _PAGE_SOFT_DIRTY;
789 return pte;
790}
791#define pte_swp_mksoft_dirty pte_mksoft_dirty
792
793static inline pte_t pte_clear_soft_dirty(pte_t pte)
794{
795 pte_val(pte) &= ~_PAGE_SOFT_DIRTY;
796 return pte;
797}
798#define pte_swp_clear_soft_dirty pte_clear_soft_dirty
799
800static inline int pmd_soft_dirty(pmd_t pmd)
801{
802 return pmd_val(pmd) & _SEGMENT_ENTRY_SOFT_DIRTY;
803}
804
805static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
806{
807 pmd_val(pmd) |= _SEGMENT_ENTRY_SOFT_DIRTY;
808 return pmd;
809}
810
811static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
812{
813 pmd_val(pmd) &= ~_SEGMENT_ENTRY_SOFT_DIRTY;
814 return pmd;
815}
816
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817/*
818 * query functions pte_write/pte_dirty/pte_young only work if
819 * pte_present() is true. Undefined behaviour if not..
820 */
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800821static inline int pte_write(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200823 return (pte_val(pte) & _PAGE_WRITE) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824}
825
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800826static inline int pte_dirty(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200828 return (pte_val(pte) & _PAGE_DIRTY) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829}
830
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800831static inline int pte_young(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832{
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200833 return (pte_val(pte) & _PAGE_YOUNG) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834}
835
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200836#define __HAVE_ARCH_PTE_UNUSED
837static inline int pte_unused(pte_t pte)
838{
839 return pte_val(pte) & _PAGE_UNUSED;
840}
841
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842/*
843 * pgd/pmd/pte modification functions
844 */
845
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200846static inline void pgd_clear(pgd_t *pgd)
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100847{
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +0200848 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1)
849 pgd_val(*pgd) = _REGION1_ENTRY_EMPTY;
850}
851
852static inline void p4d_clear(p4d_t *p4d)
853{
854 if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
855 p4d_val(*p4d) = _REGION2_ENTRY_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856}
857
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100858static inline void pud_clear(pud_t *pud)
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100859{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200860 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
861 pud_val(*pud) = _REGION3_ENTRY_EMPTY;
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100862}
Martin Schwidefsky146e4b32008-02-09 18:24:35 +0100863
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200864static inline void pmd_clear(pmd_t *pmdp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865{
Dominik Dingel54397bb2016-04-27 11:43:07 +0200866 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867}
868
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800869static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200871 pte_val(*ptep) = _PAGE_INVALID;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872}
873
874/*
875 * The following pte modification functions only work if
876 * pte_present() is true. Undefined behaviour if not..
877 */
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800878static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879{
Nick Piggin138c9022008-07-08 11:31:06 +0200880 pte_val(pte) &= _PAGE_CHG_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881 pte_val(pte) |= pgprot_val(newprot);
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200882 /*
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100883 * newprot for PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX
884 * has the invalid bit set, clear it again for readable, young pages
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200885 */
886 if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ))
887 pte_val(pte) &= ~_PAGE_INVALID;
888 /*
Martin Schwidefsky57d7f932016-03-22 10:54:24 +0100889 * newprot for PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX has the page
890 * protection bit set, clear it again for writable, dirty pages
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200891 */
Martin Schwidefskye5098612013-07-23 20:57:57 +0200892 if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE))
893 pte_val(pte) &= ~_PAGE_PROTECT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 return pte;
895}
896
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800897static inline pte_t pte_wrprotect(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200899 pte_val(pte) &= ~_PAGE_WRITE;
900 pte_val(pte) |= _PAGE_PROTECT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 return pte;
902}
903
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800904static inline pte_t pte_mkwrite(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200906 pte_val(pte) |= _PAGE_WRITE;
907 if (pte_val(pte) & _PAGE_DIRTY)
908 pte_val(pte) &= ~_PAGE_PROTECT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 return pte;
910}
911
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800912static inline pte_t pte_mkclean(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200914 pte_val(pte) &= ~_PAGE_DIRTY;
915 pte_val(pte) |= _PAGE_PROTECT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 return pte;
917}
918
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800919static inline pte_t pte_mkdirty(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920{
Martin Schwidefsky5614dd92015-04-22 14:47:42 +0200921 pte_val(pte) |= _PAGE_DIRTY | _PAGE_SOFT_DIRTY;
Martin Schwidefskye5098612013-07-23 20:57:57 +0200922 if (pte_val(pte) & _PAGE_WRITE)
923 pte_val(pte) &= ~_PAGE_PROTECT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 return pte;
925}
926
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800927static inline pte_t pte_mkold(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200929 pte_val(pte) &= ~_PAGE_YOUNG;
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200930 pte_val(pte) |= _PAGE_INVALID;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931 return pte;
932}
933
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800934static inline pte_t pte_mkyoung(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935{
Martin Schwidefsky0944fe32013-07-23 22:11:42 +0200936 pte_val(pte) |= _PAGE_YOUNG;
937 if (pte_val(pte) & _PAGE_READ)
938 pte_val(pte) &= ~_PAGE_INVALID;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 return pte;
940}
941
Nick Piggin7e675132008-04-28 02:13:00 -0700942static inline pte_t pte_mkspecial(pte_t pte)
943{
Nick Piggina08cb622008-04-28 02:13:03 -0700944 pte_val(pte) |= _PAGE_SPECIAL;
Nick Piggin7e675132008-04-28 02:13:00 -0700945 return pte;
946}
947
Heiko Carstens84afdce2010-10-25 16:10:36 +0200948#ifdef CONFIG_HUGETLB_PAGE
949static inline pte_t pte_mkhuge(pte_t pte)
950{
Martin Schwidefskye5098612013-07-23 20:57:57 +0200951 pte_val(pte) |= _PAGE_LARGE;
Heiko Carstens84afdce2010-10-25 16:10:36 +0200952 return pte;
953}
954#endif
955
Martin Schwidefsky34eeaf32016-06-14 12:38:40 +0200956#define IPTE_GLOBAL 0
957#define IPTE_LOCAL 1
958
Martin Schwidefsky118bd312016-07-26 16:53:09 +0200959#define IPTE_NODAT 0x400
Martin Schwidefsky28c807e2016-07-26 16:00:22 +0200960#define IPTE_GUEST_ASCE 0x800
Martin Schwidefsky118bd312016-07-26 16:53:09 +0200961
962static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
Martin Schwidefsky28c807e2016-07-26 16:00:22 +0200963 unsigned long opt, unsigned long asce,
964 int local)
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200965{
Martin Schwidefsky53e857f2012-09-10 13:00:09 +0200966 unsigned long pto = (unsigned long) ptep;
967
Martin Schwidefsky118bd312016-07-26 16:53:09 +0200968 if (__builtin_constant_p(opt) && opt == 0) {
969 /* Invalidation + TLB flush for the pte */
970 asm volatile(
971 " .insn rrf,0xb2210000,%[r1],%[r2],0,%[m4]"
972 : "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
973 [m4] "i" (local));
974 return;
975 }
976
977 /* Invalidate ptes with options + TLB flush of the ptes */
Martin Schwidefsky28c807e2016-07-26 16:00:22 +0200978 opt = opt | (asce & _ASCE_ORIGIN);
Martin Schwidefsky53e857f2012-09-10 13:00:09 +0200979 asm volatile(
Martin Schwidefsky118bd312016-07-26 16:53:09 +0200980 " .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
981 : [r2] "+a" (address), [r3] "+a" (opt)
982 : [r1] "a" (pto), [m4] "i" (local) : "memory");
Martin Schwidefsky53e857f2012-09-10 13:00:09 +0200983}
984
Martin Schwidefsky34eeaf32016-06-14 12:38:40 +0200985static inline void __ptep_ipte_range(unsigned long address, int nr,
986 pte_t *ptep, int local)
Martin Schwidefsky1b948d62014-04-03 13:55:01 +0200987{
988 unsigned long pto = (unsigned long) ptep;
989
Martin Schwidefsky34eeaf32016-06-14 12:38:40 +0200990 /* Invalidate a range of ptes + TLB flush of the ptes */
Heiko Carstenscfb0b242014-09-23 21:29:20 +0200991 do {
992 asm volatile(
Martin Schwidefsky34eeaf32016-06-14 12:38:40 +0200993 " .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
994 : [r2] "+a" (address), [r3] "+a" (nr)
995 : [r1] "a" (pto), [m4] "i" (local) : "memory");
Heiko Carstenscfb0b242014-09-23 21:29:20 +0200996 } while (nr != 255);
997}
998
Martin Schwidefskyba8a9222007-10-22 12:52:44 +0200999/*
1000 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1001 * both clear the TLB for the unmapped pte. The reason is that
1002 * ptep_get_and_clear is used in common code (e.g. change_pte_range)
1003 * to modify an active pte. The sequence is
1004 * 1) ptep_get_and_clear
1005 * 2) set_pte_at
1006 * 3) flush_tlb_range
1007 * On s390 the tlb needs to get flushed with the modification of the pte
1008 * if the pte is active. The only way how this can be implemented is to
1009 * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
1010 * is a nop.
1011 */
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001012pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
1013pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
1014
1015#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1016static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1017 unsigned long addr, pte_t *ptep)
1018{
1019 pte_t pte = *ptep;
1020
1021 pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
1022 return pte_young(pte);
1023}
1024
1025#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1026static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1027 unsigned long address, pte_t *ptep)
1028{
1029 return ptep_test_and_clear_young(vma, address, ptep);
1030}
1031
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001032#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001033static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001034 unsigned long addr, pte_t *ptep)
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001035{
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001036 return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001037}
1038
1039#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001040pte_t ptep_modify_prot_start(struct mm_struct *, unsigned long, pte_t *);
1041void ptep_modify_prot_commit(struct mm_struct *, unsigned long, pte_t *, pte_t);
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001042
1043#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
Martin Schwidefskyf0e47c22007-07-17 04:03:03 -07001044static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001045 unsigned long addr, pte_t *ptep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046{
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001047 return ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048}
1049
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001050/*
1051 * The batched pte unmap code uses ptep_get_and_clear_full to clear the
1052 * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
1053 * tlbs of an mm if it can guarantee that the ptes of the mm_struct
1054 * cannot be accessed while the batched unmap is running. In this case
1055 * full==1 and a simple pte_clear is enough. See tlb.h.
1056 */
1057#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1058static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001059 unsigned long addr,
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001060 pte_t *ptep, int full)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061{
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001062 if (full) {
1063 pte_t pte = *ptep;
1064 *ptep = __pte(_PAGE_INVALID);
1065 return pte;
Martin Schwidefskyd3383632013-04-17 10:53:39 +02001066 }
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001067 return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068}
1069
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001070#define __HAVE_ARCH_PTEP_SET_WRPROTECT
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001071static inline void ptep_set_wrprotect(struct mm_struct *mm,
1072 unsigned long addr, pte_t *ptep)
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001073{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001074 pte_t pte = *ptep;
1075
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001076 if (pte_write(pte))
1077 ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001078}
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001079
1080#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001081static inline int ptep_set_access_flags(struct vm_area_struct *vma,
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001082 unsigned long addr, pte_t *ptep,
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001083 pte_t entry, int dirty)
1084{
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001085 if (pte_same(*ptep, entry))
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001086 return 0;
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001087 ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001088 return 1;
1089}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090
Martin Schwidefsky1e133ab2016-03-08 11:49:57 +01001091/*
1092 * Additional functions to handle KVM guest page tables
1093 */
1094void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
1095 pte_t *ptep, pte_t entry);
1096void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
Martin Schwidefsky4be130a2016-03-08 12:12:18 +01001097void ptep_notify(struct mm_struct *mm, unsigned long addr,
1098 pte_t *ptep, unsigned long bits);
Martin Schwidefskyb2d73b22016-03-08 11:54:42 +01001099int ptep_force_prot(struct mm_struct *mm, unsigned long gaddr,
Martin Schwidefsky4be130a2016-03-08 12:12:18 +01001100 pte_t *ptep, int prot, unsigned long bit);
Martin Schwidefsky1e133ab2016-03-08 11:49:57 +01001101void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
1102 pte_t *ptep , int reset);
1103void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
Martin Schwidefsky4be130a2016-03-08 12:12:18 +01001104int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
David Hildenbranda9d23e72016-03-08 12:21:41 +01001105 pte_t *sptep, pte_t *tptep, pte_t pte);
Martin Schwidefsky4be130a2016-03-08 12:12:18 +01001106void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep);
Martin Schwidefsky1e133ab2016-03-08 11:49:57 +01001107
Janosch Frank0959e162018-07-17 13:21:22 +01001108bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address,
1109 pte_t *ptep);
Martin Schwidefsky1e133ab2016-03-08 11:49:57 +01001110int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1111 unsigned char key, bool nq);
David Hildenbrand1824c722016-05-10 09:43:11 +02001112int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1113 unsigned char key, unsigned char *oldkey,
1114 bool nq, bool mr, bool mc);
David Hildenbranda7e19ab2016-05-10 09:50:21 +02001115int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr);
David Hildenbrand154c8c12016-05-09 11:22:34 +02001116int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1117 unsigned char *key);
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001118
Claudio Imbrenda2d42f942017-04-20 10:03:45 +02001119int set_pgste_bits(struct mm_struct *mm, unsigned long addr,
1120 unsigned long bits, unsigned long value);
1121int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
1122int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1123 unsigned long *oldpte, unsigned long *oldpgste);
Janosch Frank6a376272018-07-13 11:28:22 +01001124void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
1125void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
1126void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
1127void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
Claudio Imbrenda2d42f942017-04-20 10:03:45 +02001128
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001129/*
1130 * Certain architectures need to do special things when PTEs
1131 * within a page table are directly modified. Thus, the following
1132 * hook is made available.
1133 */
1134static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
1135 pte_t *ptep, pte_t entry)
1136{
Martin Schwidefsky57d7f932016-03-22 10:54:24 +01001137 if (!MACHINE_HAS_NX)
1138 pte_val(entry) &= ~_PAGE_NOEXEC;
Christian Borntraegera8f60d12017-04-09 22:09:38 +02001139 if (pte_present(entry))
1140 pte_val(entry) &= ~_PAGE_UNUSED;
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001141 if (mm_has_pgste(mm))
Martin Schwidefsky1e133ab2016-03-08 11:49:57 +01001142 ptep_set_pte_at(mm, addr, ptep, entry);
Martin Schwidefskyebde7652016-03-08 11:08:09 +01001143 else
1144 *ptep = entry;
1145}
1146
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 * Conversion functions: convert a page and protection to a page entry,
1149 * and a page entry and page directory to the page they refer to.
1150 */
1151static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1152{
1153 pte_t __pte;
1154 pte_val(__pte) = physpage + pgprot_val(pgprot);
Martin Schwidefsky0944fe32013-07-23 22:11:42 +02001155 return pte_mkyoung(__pte);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156}
1157
Heiko Carstens2dcea572006-09-29 01:58:41 -07001158static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1159{
Heiko Carstens0b2b6e1d2006-10-04 20:02:23 +02001160 unsigned long physpage = page_to_phys(page);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001161 pte_t __pte = mk_pte_phys(physpage, pgprot);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162
Martin Schwidefskye5098612013-07-23 20:57:57 +02001163 if (pte_write(__pte) && PageDirty(page))
1164 __pte = pte_mkdirty(__pte);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001165 return __pte;
Heiko Carstens2dcea572006-09-29 01:58:41 -07001166}
1167
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +02001169#define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D-1))
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001170#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1171#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1172#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001174#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175#define pgd_offset_k(address) pgd_offset(&init_mm, address)
1176
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001177#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
1178#define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN)
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +02001179#define p4d_deref(pud) (p4d_val(pud) & _REGION_ENTRY_ORIGIN)
Martin Schwidefsky5a216a22008-02-09 18:24:36 +01001180#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN)
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001181
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +02001182static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
Martin Schwidefsky5a216a22008-02-09 18:24:36 +01001183{
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +02001184 p4d_t *p4d = (p4d_t *) pgd;
1185
1186 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1)
1187 p4d = (p4d_t *) pgd_deref(*pgd);
1188 return p4d + p4d_index(address);
1189}
1190
1191static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
1192{
1193 pud_t *pud = (pud_t *) p4d;
1194
1195 if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
1196 pud = (pud_t *) p4d_deref(*p4d);
1197 return pud + pud_index(address);
Martin Schwidefsky5a216a22008-02-09 18:24:36 +01001198}
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001199
1200static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
1201{
Martin Schwidefsky6252d702008-02-09 18:24:37 +01001202 pmd_t *pmd = (pmd_t *) pud;
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +02001203
Martin Schwidefsky6252d702008-02-09 18:24:37 +01001204 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
1205 pmd = (pmd_t *) pud_deref(*pud);
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001206 return pmd + pmd_index(address);
1207}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001209#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot))
1210#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
1211#define pte_page(x) pfn_to_page(pte_pfn(x))
1212
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001213#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
Gerald Schaeferd08de8e2016-07-04 14:47:01 +02001214#define pud_page(pud) pfn_to_page(pud_pfn(pud))
Martin Schwidefsky1aea9b32017-04-24 18:19:10 +02001215#define p4d_page(pud) pfn_to_page(p4d_pfn(p4d))
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001216
1217/* Find an entry in the lowest level page table.. */
1218#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
1219#define pte_offset_kernel(pmd, address) pte_offset(pmd,address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221#define pte_unmap(pte) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001223static inline pmd_t pmd_wrprotect(pmd_t pmd)
1224{
1225 pmd_val(pmd) &= ~_SEGMENT_ENTRY_WRITE;
1226 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
1227 return pmd;
1228}
1229
1230static inline pmd_t pmd_mkwrite(pmd_t pmd)
1231{
1232 pmd_val(pmd) |= _SEGMENT_ENTRY_WRITE;
1233 if (pmd_large(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY))
1234 return pmd;
1235 pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT;
1236 return pmd;
1237}
1238
1239static inline pmd_t pmd_mkclean(pmd_t pmd)
1240{
1241 if (pmd_large(pmd)) {
1242 pmd_val(pmd) &= ~_SEGMENT_ENTRY_DIRTY;
1243 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
1244 }
1245 return pmd;
1246}
1247
1248static inline pmd_t pmd_mkdirty(pmd_t pmd)
1249{
1250 if (pmd_large(pmd)) {
Martin Schwidefsky5614dd92015-04-22 14:47:42 +02001251 pmd_val(pmd) |= _SEGMENT_ENTRY_DIRTY |
1252 _SEGMENT_ENTRY_SOFT_DIRTY;
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001253 if (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE)
1254 pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT;
1255 }
1256 return pmd;
1257}
1258
Heiko Carstens9e20b4d2016-05-10 10:34:47 +02001259static inline pud_t pud_wrprotect(pud_t pud)
1260{
1261 pud_val(pud) &= ~_REGION3_ENTRY_WRITE;
1262 pud_val(pud) |= _REGION_ENTRY_PROTECT;
1263 return pud;
1264}
1265
1266static inline pud_t pud_mkwrite(pud_t pud)
1267{
1268 pud_val(pud) |= _REGION3_ENTRY_WRITE;
1269 if (pud_large(pud) && !(pud_val(pud) & _REGION3_ENTRY_DIRTY))
1270 return pud;
1271 pud_val(pud) &= ~_REGION_ENTRY_PROTECT;
1272 return pud;
1273}
1274
1275static inline pud_t pud_mkclean(pud_t pud)
1276{
1277 if (pud_large(pud)) {
1278 pud_val(pud) &= ~_REGION3_ENTRY_DIRTY;
1279 pud_val(pud) |= _REGION_ENTRY_PROTECT;
1280 }
1281 return pud;
1282}
1283
1284static inline pud_t pud_mkdirty(pud_t pud)
1285{
1286 if (pud_large(pud)) {
1287 pud_val(pud) |= _REGION3_ENTRY_DIRTY |
1288 _REGION3_ENTRY_SOFT_DIRTY;
1289 if (pud_val(pud) & _REGION3_ENTRY_WRITE)
1290 pud_val(pud) &= ~_REGION_ENTRY_PROTECT;
1291 }
1292 return pud;
1293}
1294
1295#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
1296static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1297{
1298 /*
Martin Schwidefsky57d7f932016-03-22 10:54:24 +01001299 * pgprot is PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW or PAGE_RWX
1300 * (see __Pxxx / __Sxxx). Convert to segment table entry format.
Heiko Carstens9e20b4d2016-05-10 10:34:47 +02001301 */
1302 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1303 return pgprot_val(SEGMENT_NONE);
Martin Schwidefsky57d7f932016-03-22 10:54:24 +01001304 if (pgprot_val(pgprot) == pgprot_val(PAGE_RO))
1305 return pgprot_val(SEGMENT_RO);
1306 if (pgprot_val(pgprot) == pgprot_val(PAGE_RX))
1307 return pgprot_val(SEGMENT_RX);
1308 if (pgprot_val(pgprot) == pgprot_val(PAGE_RW))
1309 return pgprot_val(SEGMENT_RW);
1310 return pgprot_val(SEGMENT_RWX);
Heiko Carstens9e20b4d2016-05-10 10:34:47 +02001311}
1312
Martin Schwidefsky0944fe32013-07-23 22:11:42 +02001313static inline pmd_t pmd_mkyoung(pmd_t pmd)
1314{
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001315 if (pmd_large(pmd)) {
Martin Schwidefsky0944fe32013-07-23 22:11:42 +02001316 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG;
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001317 if (pmd_val(pmd) & _SEGMENT_ENTRY_READ)
1318 pmd_val(pmd) &= ~_SEGMENT_ENTRY_INVALID;
Martin Schwidefsky0944fe32013-07-23 22:11:42 +02001319 }
Martin Schwidefsky0944fe32013-07-23 22:11:42 +02001320 return pmd;
1321}
1322
1323static inline pmd_t pmd_mkold(pmd_t pmd)
1324{
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001325 if (pmd_large(pmd)) {
Martin Schwidefsky0944fe32013-07-23 22:11:42 +02001326 pmd_val(pmd) &= ~_SEGMENT_ENTRY_YOUNG;
1327 pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID;
1328 }
Martin Schwidefsky0944fe32013-07-23 22:11:42 +02001329 return pmd;
1330}
1331
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001332static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1333{
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001334 if (pmd_large(pmd)) {
1335 pmd_val(pmd) &= _SEGMENT_ENTRY_ORIGIN_LARGE |
1336 _SEGMENT_ENTRY_DIRTY | _SEGMENT_ENTRY_YOUNG |
Kirill A. Shutemovfecffad2016-01-15 16:53:24 -08001337 _SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_SOFT_DIRTY;
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001338 pmd_val(pmd) |= massage_pgprot_pmd(newprot);
1339 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY))
1340 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
1341 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG))
1342 pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID;
1343 return pmd;
1344 }
1345 pmd_val(pmd) &= _SEGMENT_ENTRY_ORIGIN;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001346 pmd_val(pmd) |= massage_pgprot_pmd(newprot);
1347 return pmd;
1348}
1349
Gerald Schaefer106c9922013-04-29 15:07:23 -07001350static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001351{
Gerald Schaefer106c9922013-04-29 15:07:23 -07001352 pmd_t __pmd;
1353 pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001354 return __pmd;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001355}
1356
Gerald Schaefer106c9922013-04-29 15:07:23 -07001357#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */
1358
Martin Schwidefsky1b948d62014-04-03 13:55:01 +02001359static inline void __pmdp_csp(pmd_t *pmdp)
1360{
Heiko Carstens4ccccc52016-05-14 10:46:33 +02001361 csp((unsigned int *)pmdp + 1, pmd_val(*pmdp),
1362 pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
Martin Schwidefsky1b948d62014-04-03 13:55:01 +02001363}
1364
Martin Schwidefsky47e4d852016-06-14 12:41:35 +02001365#define IDTE_GLOBAL 0
1366#define IDTE_LOCAL 1
1367
Martin Schwidefsky118bd312016-07-26 16:53:09 +02001368#define IDTE_PTOA 0x0800
1369#define IDTE_NODAT 0x1000
Martin Schwidefsky28c807e2016-07-26 16:00:22 +02001370#define IDTE_GUEST_ASCE 0x2000
Martin Schwidefsky118bd312016-07-26 16:53:09 +02001371
1372static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
Martin Schwidefsky28c807e2016-07-26 16:00:22 +02001373 unsigned long opt, unsigned long asce,
1374 int local)
Martin Schwidefsky1b948d62014-04-03 13:55:01 +02001375{
1376 unsigned long sto;
1377
Martin Schwidefsky118bd312016-07-26 16:53:09 +02001378 sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t);
Martin Schwidefsky28c807e2016-07-26 16:00:22 +02001379 if (__builtin_constant_p(opt) && opt == 0) {
1380 /* flush without guest asce */
1381 asm volatile(
1382 " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
1383 : "+m" (*pmdp)
1384 : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
1385 [m4] "i" (local)
1386 : "cc" );
1387 } else {
1388 /* flush with guest asce */
1389 asm volatile(
1390 " .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
1391 : "+m" (*pmdp)
1392 : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
1393 [r3] "a" (asce), [m4] "i" (local)
1394 : "cc" );
1395 }
Martin Schwidefsky1b948d62014-04-03 13:55:01 +02001396}
1397
Martin Schwidefsky118bd312016-07-26 16:53:09 +02001398static inline void __pudp_idte(unsigned long addr, pud_t *pudp,
Martin Schwidefsky28c807e2016-07-26 16:00:22 +02001399 unsigned long opt, unsigned long asce,
1400 int local)
Gerald Schaeferd08de8e2016-07-04 14:47:01 +02001401{
1402 unsigned long r3o;
1403
Martin Schwidefsky118bd312016-07-26 16:53:09 +02001404 r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t);
Gerald Schaeferd08de8e2016-07-04 14:47:01 +02001405 r3o |= _ASCE_TYPE_REGION3;
Martin Schwidefsky28c807e2016-07-26 16:00:22 +02001406 if (__builtin_constant_p(opt) && opt == 0) {
1407 /* flush without guest asce */
1408 asm volatile(
1409 " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
1410 : "+m" (*pudp)
1411 : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
1412 [m4] "i" (local)
1413 : "cc");
1414 } else {
1415 /* flush with guest asce */
1416 asm volatile(
1417 " .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
1418 : "+m" (*pudp)
1419 : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
1420 [r3] "a" (asce), [m4] "i" (local)
1421 : "cc" );
1422 }
Gerald Schaeferd08de8e2016-07-04 14:47:01 +02001423}
1424
Martin Schwidefsky227be792016-03-08 11:09:25 +01001425pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1426pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
Gerald Schaeferd08de8e2016-07-04 14:47:01 +02001427pud_t pudp_xchg_direct(struct mm_struct *, unsigned long, pud_t *, pud_t);
Martin Schwidefsky3eabaee2013-07-26 15:04:02 +02001428
Gerald Schaefer106c9922013-04-29 15:07:23 -07001429#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1430
1431#define __HAVE_ARCH_PGTABLE_DEPOSIT
Martin Schwidefsky227be792016-03-08 11:09:25 +01001432void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1433 pgtable_t pgtable);
Gerald Schaefer106c9922013-04-29 15:07:23 -07001434
1435#define __HAVE_ARCH_PGTABLE_WITHDRAW
Martin Schwidefsky227be792016-03-08 11:09:25 +01001436pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1437
1438#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1439static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1440 unsigned long addr, pmd_t *pmdp,
1441 pmd_t entry, int dirty)
1442{
1443 VM_BUG_ON(addr & ~HPAGE_MASK);
1444
1445 entry = pmd_mkyoung(entry);
1446 if (dirty)
1447 entry = pmd_mkdirty(entry);
1448 if (pmd_val(*pmdp) == pmd_val(entry))
1449 return 0;
1450 pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);
1451 return 1;
1452}
1453
1454#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1455static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1456 unsigned long addr, pmd_t *pmdp)
1457{
1458 pmd_t pmd = *pmdp;
1459
1460 pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));
1461 return pmd_young(pmd);
1462}
1463
1464#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
1465static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
1466 unsigned long addr, pmd_t *pmdp)
1467{
1468 VM_BUG_ON(addr & ~HPAGE_MASK);
1469 return pmdp_test_and_clear_young(vma, addr, pmdp);
1470}
Gerald Schaefer106c9922013-04-29 15:07:23 -07001471
Gerald Schaefer106c9922013-04-29 15:07:23 -07001472static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1473 pmd_t *pmdp, pmd_t entry)
1474{
Martin Schwidefsky57d7f932016-03-22 10:54:24 +01001475 if (!MACHINE_HAS_NX)
1476 pmd_val(entry) &= ~_SEGMENT_ENTRY_NOEXEC;
Gerald Schaefer106c9922013-04-29 15:07:23 -07001477 *pmdp = entry;
1478}
1479
1480static inline pmd_t pmd_mkhuge(pmd_t pmd)
1481{
1482 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
Martin Schwidefsky152125b2014-07-24 11:03:41 +02001483 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG;
1484 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001485 return pmd;
1486}
1487
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001488#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1489static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
Martin Schwidefsky227be792016-03-08 11:09:25 +01001490 unsigned long addr, pmd_t *pmdp)
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001491{
Dominik Dingel54397bb2016-04-27 11:43:07 +02001492 return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001493}
1494
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001495#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
1496static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm,
Martin Schwidefsky227be792016-03-08 11:09:25 +01001497 unsigned long addr,
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001498 pmd_t *pmdp, int full)
Martin Schwidefskyfcbe08d62014-10-24 10:52:29 +02001499{
Martin Schwidefsky227be792016-03-08 11:09:25 +01001500 if (full) {
1501 pmd_t pmd = *pmdp;
Dominik Dingel54397bb2016-04-27 11:43:07 +02001502 *pmdp = __pmd(_SEGMENT_ENTRY_EMPTY);
Martin Schwidefsky227be792016-03-08 11:09:25 +01001503 return pmd;
1504 }
Dominik Dingel54397bb2016-04-27 11:43:07 +02001505 return pmdp_xchg_lazy(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
Martin Schwidefskyfcbe08d62014-10-24 10:52:29 +02001506}
1507
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001508#define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
1509static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
Martin Schwidefsky227be792016-03-08 11:09:25 +01001510 unsigned long addr, pmd_t *pmdp)
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001511{
Martin Schwidefsky227be792016-03-08 11:09:25 +01001512 return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001513}
1514
1515#define __HAVE_ARCH_PMDP_INVALIDATE
Martin Schwidefsky9c4563f2018-01-31 16:18:05 -08001516static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
Martin Schwidefsky227be792016-03-08 11:09:25 +01001517 unsigned long addr, pmd_t *pmdp)
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001518{
Gerald Schaefer91c575b2017-09-18 16:10:35 +02001519 pmd_t pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1520
Martin Schwidefsky9c4563f2018-01-31 16:18:05 -08001521 return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001522}
1523
Gerald Schaeferbe328652013-01-21 16:48:07 +01001524#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1525static inline void pmdp_set_wrprotect(struct mm_struct *mm,
Martin Schwidefsky227be792016-03-08 11:09:25 +01001526 unsigned long addr, pmd_t *pmdp)
Gerald Schaeferbe328652013-01-21 16:48:07 +01001527{
1528 pmd_t pmd = *pmdp;
1529
Martin Schwidefsky227be792016-03-08 11:09:25 +01001530 if (pmd_write(pmd))
1531 pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd));
Gerald Schaeferbe328652013-01-21 16:48:07 +01001532}
1533
Aneesh Kumar K.Vf28b6ff2015-06-24 16:57:42 -07001534static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1535 unsigned long address,
1536 pmd_t *pmdp)
1537{
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001538 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
Aneesh Kumar K.Vf28b6ff2015-06-24 16:57:42 -07001539}
1540#define pmdp_collapse_flush pmdp_collapse_flush
1541
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001542#define pfn_pmd(pfn, pgprot) mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
1543#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
1544
1545static inline int pmd_trans_huge(pmd_t pmd)
1546{
1547 return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
1548}
1549
Hugh Dickinsfd8cfd32016-05-19 17:13:00 -07001550#define has_transparent_hugepage has_transparent_hugepage
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001551static inline int has_transparent_hugepage(void)
1552{
Heiko Carstens466178f2017-02-13 15:11:15 +01001553 return MACHINE_HAS_EDAT1 ? 1 : 0;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001554}
Gerald Schaefer75077af2012-10-08 16:30:15 -07001555#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1556
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558 * 64 bit swap entry format:
1559 * A page-table entry has some bits we have to treat in a special way.
Geert Uytterhoeven4e0a6412015-05-21 14:00:47 +02001560 * Bits 52 and bit 55 have to be zero, otherwise a specification
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561 * exception will occur instead of a page translation exception. The
Geert Uytterhoeven4e0a6412015-05-21 14:00:47 +02001562 * specification exception has the bad habit not to store necessary
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563 * information in the lowcore.
Martin Schwidefskya1c843b2015-04-22 13:55:59 +02001564 * Bits 54 and 63 are used to indicate the page type.
1565 * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
1566 * This leaves the bits 0-51 and bits 56-62 to store type and offset.
1567 * We use the 5 bits from 57-61 for the type and the 52 bits from 0-51
1568 * for the offset.
1569 * | offset |01100|type |00|
1570 * |0000000000111111111122222222223333333333444444444455|55555|55566|66|
1571 * |0123456789012345678901234567890123456789012345678901|23456|78901|23|
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572 */
Heiko Carstens5a79859a2015-02-12 13:08:27 +01001573
Martin Schwidefskya1c843b2015-04-22 13:55:59 +02001574#define __SWP_OFFSET_MASK ((1UL << 52) - 1)
1575#define __SWP_OFFSET_SHIFT 12
1576#define __SWP_TYPE_MASK ((1UL << 5) - 1)
1577#define __SWP_TYPE_SHIFT 2
Heiko Carstens5a79859a2015-02-12 13:08:27 +01001578
Adrian Bunk4448aaf2005-11-08 21:34:42 -08001579static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580{
1581 pte_t pte;
Martin Schwidefskya1c843b2015-04-22 13:55:59 +02001582
1583 pte_val(pte) = _PAGE_INVALID | _PAGE_PROTECT;
1584 pte_val(pte) |= (offset & __SWP_OFFSET_MASK) << __SWP_OFFSET_SHIFT;
1585 pte_val(pte) |= (type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586 return pte;
1587}
1588
Martin Schwidefskya1c843b2015-04-22 13:55:59 +02001589static inline unsigned long __swp_type(swp_entry_t entry)
1590{
1591 return (entry.val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK;
1592}
1593
1594static inline unsigned long __swp_offset(swp_entry_t entry)
1595{
1596 return (entry.val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK;
1597}
1598
1599static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset)
1600{
1601 return (swp_entry_t) { pte_val(mk_swap_pte(type, offset)) };
1602}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603
1604#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1605#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
1606
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607#define kern_addr_valid(addr) (1)
1608
Heiko Carstens17f34582008-04-30 13:38:47 +02001609extern int vmem_add_mapping(unsigned long start, unsigned long size);
1610extern int vmem_remove_mapping(unsigned long start, unsigned long size);
Carsten Otte402b0862008-03-25 18:47:10 +01001611extern int s390_enable_sie(void);
Dominik Dingel3ac8e382014-10-23 12:09:17 +02001612extern int s390_enable_skey(void);
Dominik Dingela13cff32014-10-23 12:07:14 +02001613extern void s390_reset_cmma(struct mm_struct *mm);
Heiko Carstensf4eb07c2006-12-08 15:56:07 +01001614
Martin Schwidefsky1f6b83e2015-01-14 17:51:17 +01001615/* s390 has a private copy of get unmapped area to deal with cache synonyms */
1616#define HAVE_ARCH_UNMAPPED_AREA
1617#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1618
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619/*
1620 * No page table caches to initialise
1621 */
Heiko Carstens765a0ca2013-03-23 10:29:01 +01001622static inline void pgtable_cache_init(void) { }
1623static inline void check_pgt_cache(void) { }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625#include <asm-generic/pgtable.h>
1626
1627#endif /* _S390_PAGE_H */