blob: 5ea05879a0a94a6593aebb2a0c494abf0a1ffbc4 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002#ifndef _LINUX_HUGETLB_H
3#define _LINUX_HUGETLB_H
4
Linus Torvaldsbe93d8c2011-05-26 12:03:50 -07005#include <linux/mm_types.h>
Sasha Levin309381fea2014-01-23 15:52:54 -08006#include <linux/mmdebug.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +04007#include <linux/fs.h>
Naoya Horiguchi8edf3442010-05-28 09:29:15 +09008#include <linux/hugetlb_inline.h>
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -07009#include <linux/cgroup.h>
Joonsoo Kim9119a412014-04-03 14:47:25 -070010#include <linux/list.h>
11#include <linux/kref.h>
Dan Williams888cdbc2016-01-15 16:56:32 -080012#include <asm/pgtable.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040013
Andrew Mortone9ea0e22009-09-24 14:47:45 -070014struct ctl_table;
15struct user_struct;
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -070016struct mmu_gather;
Andrew Mortone9ea0e22009-09-24 14:47:45 -070017
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070018#ifndef is_hugepd
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070019typedef struct { unsigned long pd; } hugepd_t;
20#define is_hugepd(hugepd) (0)
21#define __hugepd(x) ((hugepd_t) { (x) })
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070022#endif
23
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#ifdef CONFIG_HUGETLB_PAGE
25
26#include <linux/mempolicy.h>
Adam Litke516dffd2007-03-01 15:46:08 -080027#include <linux/shm.h>
David Gibson63551ae2005-06-21 17:14:44 -070028#include <asm/tlbflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
David Gibson90481622012-03-21 16:34:12 -070030struct hugepage_subpool {
31 spinlock_t lock;
32 long count;
Mike Kravetzc6a91822015-04-15 16:13:36 -070033 long max_hpages; /* Maximum huge pages or -1 if no maximum. */
34 long used_hpages; /* Used count against maximum, includes */
35 /* both alloced and reserved pages. */
36 struct hstate *hstate;
37 long min_hpages; /* Minimum huge pages or -1 if no minimum. */
38 long rsv_hpages; /* Pages reserved against global pool to */
39 /* sasitfy minimum size. */
David Gibson90481622012-03-21 16:34:12 -070040};
41
Joonsoo Kim9119a412014-04-03 14:47:25 -070042struct resv_map {
43 struct kref refs;
Davidlohr Bueso7b24d862014-04-03 14:47:27 -070044 spinlock_t lock;
Joonsoo Kim9119a412014-04-03 14:47:25 -070045 struct list_head regions;
Mike Kravetz5e911372015-09-08 15:01:28 -070046 long adds_in_progress;
47 struct list_head region_cache;
48 long region_cache_count;
Mina Almasrye9fe92a2020-04-01 21:11:21 -070049#ifdef CONFIG_CGROUP_HUGETLB
50 /*
51 * On private mappings, the counter to uncharge reservations is stored
52 * here. If these fields are 0, then either the mapping is shared, or
53 * cgroup accounting is disabled for this resv_map.
54 */
55 struct page_counter *reservation_counter;
56 unsigned long pages_per_hpage;
57 struct cgroup_subsys_state *css;
58#endif
Joonsoo Kim9119a412014-04-03 14:47:25 -070059};
Mina Almasry075a61d2020-04-01 21:11:28 -070060
61/*
62 * Region tracking -- allows tracking of reservations and instantiated pages
63 * across the pages in a mapping.
64 *
65 * The region data structures are embedded into a resv_map and protected
66 * by a resv_map's lock. The set of regions within the resv_map represent
67 * reservations for huge pages, or huge pages that have already been
68 * instantiated within the map. The from and to elements are huge page
69 * indicies into the associated mapping. from indicates the starting index
70 * of the region. to represents the first index past the end of the region.
71 *
72 * For example, a file region structure with from == 0 and to == 4 represents
73 * four huge pages in a mapping. It is important to note that the to element
74 * represents the first element past the end of the region. This is used in
75 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
76 *
77 * Interval notation of the form [from, to) will be used to indicate that
78 * the endpoint from is inclusive and to is exclusive.
79 */
80struct file_region {
81 struct list_head link;
82 long from;
83 long to;
84#ifdef CONFIG_CGROUP_HUGETLB
85 /*
86 * On shared mappings, each reserved region appears as a struct
87 * file_region in resv_map. These fields hold the info needed to
88 * uncharge each reservation.
89 */
90 struct page_counter *reservation_counter;
91 struct cgroup_subsys_state *css;
92#endif
93};
94
Joonsoo Kim9119a412014-04-03 14:47:25 -070095extern struct resv_map *resv_map_alloc(void);
96void resv_map_release(struct kref *ref);
97
Aneesh Kumar K.Vc3f38a32012-07-31 16:42:10 -070098extern spinlock_t hugetlb_lock;
99extern int hugetlb_max_hstate __read_mostly;
100#define for_each_hstate(h) \
101 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
102
Mike Kravetz7ca02d0a2015-04-15 16:13:42 -0700103struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
104 long min_hpages);
David Gibson90481622012-03-21 16:34:12 -0700105void hugepage_put_subpool(struct hugepage_subpool *spool);
106
Mel Gormana1e78772008-07-23 21:27:23 -0700107void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
Alexey Dobriyan8d65af72009-09-23 15:57:19 -0700108int hugetlb_sysctl_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
109int hugetlb_overcommit_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
110int hugetlb_treat_movable_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
Lee Schermerhorn06808b02009-12-14 17:58:21 -0800111
112#ifdef CONFIG_NUMA
113int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int,
114 void __user *, size_t *, loff_t *);
115#endif
116
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
Michel Lespinasse28a35712013-02-22 16:35:55 -0800118long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
119 struct page **, struct vm_area_struct **,
Andrea Arcangeli87ffc112017-02-22 15:43:13 -0800120 unsigned long *, unsigned long *, long, unsigned int,
121 int *);
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700122void unmap_hugepage_range(struct vm_area_struct *,
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700123 unsigned long, unsigned long, struct page *);
Mel Gormand8333522012-07-31 16:46:20 -0700124void __unmap_hugepage_range_final(struct mmu_gather *tlb,
125 struct vm_area_struct *vma,
126 unsigned long start, unsigned long end,
127 struct page *ref_page);
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700128void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
129 unsigned long start, unsigned long end,
130 struct page *ref_page);
Alexey Dobriyane1759c22008-10-15 23:50:22 +0400131void hugetlb_report_meminfo(struct seq_file *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132int hugetlb_report_node_meminfo(int, char *);
David Rientjes949f7ec2013-04-29 15:07:48 -0700133void hugetlb_show_meminfo(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134unsigned long hugetlb_total_pages(void);
Souptick Joarder2b740302018-08-23 17:01:36 -0700135vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
Hugh Dickins788c7df2009-06-23 13:49:05 +0100136 unsigned long address, unsigned int flags);
Mike Kravetz8fb5deb2017-02-22 15:42:52 -0800137int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte,
138 struct vm_area_struct *dst_vma,
139 unsigned long dst_addr,
140 unsigned long src_addr,
141 struct page **pagep);
Mel Gormana1e78772008-07-23 21:27:23 -0700142int hugetlb_reserve_pages(struct inode *inode, long from, long to,
Mel Gorman5a6fe122009-02-10 14:02:27 +0000143 struct vm_area_struct *vma,
KOSAKI Motohiroca16d142011-05-26 19:16:19 +0900144 vm_flags_t vm_flags);
Mike Kravetzb5cec282015-09-08 15:01:41 -0700145long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
146 long freed);
Naoya Horiguchi31caf662013-09-11 14:21:59 -0700147bool isolate_huge_page(struct page *page, struct list_head *list);
148void putback_active_hugepage(struct page *page);
Michal Hockoab5ac902018-01-31 16:20:48 -0800149void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason);
Atsushi Kumagai8f1d26d2014-07-30 16:08:39 -0700150void free_huge_page(struct page *page);
zhong jiang72e29362016-10-07 17:02:01 -0700151void hugetlb_fix_reserve_counts(struct inode *inode);
Mike Kravetzc672c7f2015-09-08 15:01:35 -0700152extern struct mutex *hugetlb_fault_mutex_table;
Wei Yang188b04a2019-11-30 17:57:02 -0800153u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154
Steve Capper3212b532013-04-23 12:35:02 +0100155pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
Steve Capper3212b532013-04-23 12:35:02 +0100156
Mike Kravetzc0d03812020-04-01 21:11:05 -0700157struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage);
158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159extern int sysctl_hugetlb_shm_group;
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700160extern struct list_head huge_boot_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
David Gibson63551ae2005-06-21 17:14:44 -0700162/* arch callbacks */
163
Andi Kleena5516432008-07-23 21:27:41 -0700164pte_t *huge_pte_alloc(struct mm_struct *mm,
165 unsigned long addr, unsigned long sz);
Punit Agrawal7868a202017-07-06 15:39:42 -0700166pte_t *huge_pte_offset(struct mm_struct *mm,
167 unsigned long addr, unsigned long sz);
Chen, Kenneth W39dde652006-12-06 20:32:03 -0800168int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep);
Mike Kravetz017b1662018-10-05 15:51:29 -0700169void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
170 unsigned long *start, unsigned long *end);
David Gibson63551ae2005-06-21 17:14:44 -0700171struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
172 int write);
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700173struct page *follow_huge_pd(struct vm_area_struct *vma,
174 unsigned long address, hugepd_t hpd,
175 int flags, int pdshift);
David Gibson63551ae2005-06-21 17:14:44 -0700176struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
Naoya Horiguchie66f17f2015-02-11 15:25:22 -0800177 pmd_t *pmd, int flags);
Andi Kleenceb86872008-07-23 21:27:50 -0700178struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
Naoya Horiguchie66f17f2015-02-11 15:25:22 -0800179 pud_t *pud, int flags);
Anshuman Khandualfaaa5b62017-07-06 15:38:50 -0700180struct page *follow_huge_pgd(struct mm_struct *mm, unsigned long address,
181 pgd_t *pgd, int flags);
182
David Gibson63551ae2005-06-21 17:14:44 -0700183int pmd_huge(pmd_t pmd);
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300184int pud_huge(pud_t pud);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100185unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800186 unsigned long address, unsigned long end, pgprot_t newprot);
David Gibson63551ae2005-06-21 17:14:44 -0700187
Aneesh Kumar K.Vd5ed7442017-07-06 15:38:47 -0700188bool is_hugetlb_entry_migration(pte_t pte);
Michal Hockoab5ac902018-01-31 16:20:48 -0800189
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190#else /* !CONFIG_HUGETLB_PAGE */
191
Mel Gormana1e78772008-07-23 21:27:23 -0700192static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
193{
194}
195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196static inline unsigned long hugetlb_total_pages(void)
197{
198 return 0;
199}
200
Mike Kravetzc0d03812020-04-01 21:11:05 -0700201static inline struct address_space *hugetlb_page_mapping_lock_write(
202 struct page *hpage)
203{
204 return NULL;
205}
206
Mike Kravetz017b1662018-10-05 15:51:29 -0700207static inline int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr,
208 pte_t *ptep)
209{
210 return 0;
211}
212
213static inline void adjust_range_if_pmd_sharing_possible(
214 struct vm_area_struct *vma,
215 unsigned long *start, unsigned long *end)
216{
217}
218
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800219static inline long follow_hugetlb_page(struct mm_struct *mm,
220 struct vm_area_struct *vma, struct page **pages,
221 struct vm_area_struct **vmas, unsigned long *position,
222 unsigned long *nr_pages, long i, unsigned int flags,
223 int *nonblocking)
224{
225 BUG();
226 return 0;
227}
228
229static inline struct page *follow_huge_addr(struct mm_struct *mm,
230 unsigned long address, int write)
231{
232 return ERR_PTR(-EINVAL);
233}
234
235static inline int copy_hugetlb_page_range(struct mm_struct *dst,
236 struct mm_struct *src, struct vm_area_struct *vma)
237{
238 BUG();
239 return 0;
240}
241
Alexey Dobriyane1759c22008-10-15 23:50:22 +0400242static inline void hugetlb_report_meminfo(struct seq_file *m)
243{
244}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800245
246static inline int hugetlb_report_node_meminfo(int nid, char *buf)
247{
248 return 0;
249}
250
David Rientjes949f7ec2013-04-29 15:07:48 -0700251static inline void hugetlb_show_meminfo(void)
252{
253}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800254
255static inline struct page *follow_huge_pd(struct vm_area_struct *vma,
256 unsigned long address, hugepd_t hpd, int flags,
257 int pdshift)
258{
259 return NULL;
260}
261
262static inline struct page *follow_huge_pmd(struct mm_struct *mm,
263 unsigned long address, pmd_t *pmd, int flags)
264{
265 return NULL;
266}
267
268static inline struct page *follow_huge_pud(struct mm_struct *mm,
269 unsigned long address, pud_t *pud, int flags)
270{
271 return NULL;
272}
273
274static inline struct page *follow_huge_pgd(struct mm_struct *mm,
275 unsigned long address, pgd_t *pgd, int flags)
276{
277 return NULL;
278}
279
280static inline int prepare_hugepage_range(struct file *file,
281 unsigned long addr, unsigned long len)
282{
283 return -EINVAL;
284}
285
286static inline int pmd_huge(pmd_t pmd)
287{
288 return 0;
289}
290
291static inline int pud_huge(pud_t pud)
292{
293 return 0;
294}
295
296static inline int is_hugepage_only_range(struct mm_struct *mm,
297 unsigned long addr, unsigned long len)
298{
299 return 0;
300}
301
302static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
303 unsigned long addr, unsigned long end,
304 unsigned long floor, unsigned long ceiling)
305{
306 BUG();
307}
308
309static inline int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
310 pte_t *dst_pte,
311 struct vm_area_struct *dst_vma,
312 unsigned long dst_addr,
313 unsigned long src_addr,
314 struct page **pagep)
315{
316 BUG();
317 return 0;
318}
319
320static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
321 unsigned long sz)
322{
323 return NULL;
324}
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700325
Naoya Horiguchif40386a2013-12-12 17:12:19 -0800326static inline bool isolate_huge_page(struct page *page, struct list_head *list)
327{
328 return false;
329}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800331static inline void putback_active_hugepage(struct page *page)
332{
333}
334
335static inline void move_hugetlb_state(struct page *oldpage,
336 struct page *newpage, int reason)
337{
338}
339
340static inline unsigned long hugetlb_change_protection(
341 struct vm_area_struct *vma, unsigned long address,
342 unsigned long end, pgprot_t newprot)
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100343{
344 return 0;
345}
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800346
Mel Gormand8333522012-07-31 16:46:20 -0700347static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
348 struct vm_area_struct *vma, unsigned long start,
349 unsigned long end, struct page *ref_page)
350{
351 BUG();
352}
353
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700354static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
355 struct vm_area_struct *vma, unsigned long start,
356 unsigned long end, struct page *ref_page)
357{
358 BUG();
359}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800360
Souptick Joardera953e772019-03-28 20:43:51 -0700361static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800362 struct vm_area_struct *vma, unsigned long address,
363 unsigned int flags)
Souptick Joardera953e772019-03-28 20:43:51 -0700364{
365 BUG();
366 return 0;
367}
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700368
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369#endif /* !CONFIG_HUGETLB_PAGE */
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +0530370/*
371 * hugepages at page global directory. If arch support
372 * hugepages at pgd level, they need to define this.
373 */
374#ifndef pgd_huge
375#define pgd_huge(x) 0
376#endif
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300377#ifndef p4d_huge
378#define p4d_huge(x) 0
379#endif
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +0530380
381#ifndef pgd_write
382static inline int pgd_write(pgd_t pgd)
383{
384 BUG();
385 return 0;
386}
387#endif
388
Eric B Munson4e527802009-09-21 17:03:47 -0700389#define HUGETLB_ANON_FILE "anon_hugepage"
390
Eric B Munson6bfde052009-09-21 17:03:43 -0700391enum {
392 /*
393 * The file will be used as an shm file so shmfs accounting rules
394 * apply
395 */
396 HUGETLB_SHMFS_INODE = 1,
Eric B Munson4e527802009-09-21 17:03:47 -0700397 /*
398 * The file is being created on the internal vfs mount and shmfs
399 * accounting rules do not apply
400 */
401 HUGETLB_ANONHUGE_INODE = 2,
Eric B Munson6bfde052009-09-21 17:03:43 -0700402};
403
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404#ifdef CONFIG_HUGETLBFS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405struct hugetlbfs_sb_info {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 long max_inodes; /* inodes allowed */
407 long free_inodes; /* inodes free */
408 spinlock_t stat_lock;
Andi Kleena137e1c2008-07-23 21:27:43 -0700409 struct hstate *hstate;
David Gibson90481622012-03-21 16:34:12 -0700410 struct hugepage_subpool *spool;
David Howells4a252202017-07-05 16:24:18 +0100411 kuid_t uid;
412 kgid_t gid;
413 umode_t mode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414};
415
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
417{
418 return sb->s_fs_info;
419}
420
Marc-André Lureauda14c1e2018-01-31 16:19:22 -0800421struct hugetlbfs_inode_info {
422 struct shared_policy policy;
423 struct inode vfs_inode;
Marc-André Lureauff62a342018-01-31 16:19:25 -0800424 unsigned int seals;
Marc-André Lureauda14c1e2018-01-31 16:19:22 -0800425};
426
427static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
428{
429 return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
430}
431
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -0800432extern const struct file_operations hugetlbfs_file_operations;
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +0400433extern const struct vm_operations_struct hugetlb_vm_ops;
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700434struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
Andi Kleen42d73952012-12-11 16:01:34 -0800435 struct user_struct **user, int creat_flags,
436 int page_size_log);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437
Yaowei Bai719ff3212016-01-14 15:18:51 -0800438static inline bool is_file_hugepages(struct file *file)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439{
Adam Litke516dffd2007-03-01 15:46:08 -0800440 if (file->f_op == &hugetlbfs_file_operations)
Yaowei Bai719ff3212016-01-14 15:18:51 -0800441 return true;
Adam Litke516dffd2007-03-01 15:46:08 -0800442
Yaowei Bai719ff3212016-01-14 15:18:51 -0800443 return is_file_shm_hugepages(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444}
445
Christophe Leroybb297bb2020-04-01 21:11:54 -0700446static inline struct hstate *hstate_inode(struct inode *i)
447{
448 return HUGETLBFS_SB(i->i_sb)->hstate;
449}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450#else /* !CONFIG_HUGETLBFS */
451
Yaowei Bai719ff3212016-01-14 15:18:51 -0800452#define is_file_hugepages(file) false
Steven Truelove40716e22012-03-21 16:34:14 -0700453static inline struct file *
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700454hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
455 struct user_struct **user, int creat_flags,
Andi Kleen42d73952012-12-11 16:01:34 -0800456 int page_size_log)
Andrew Mortone9ea0e22009-09-24 14:47:45 -0700457{
458 return ERR_PTR(-ENOSYS);
459}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460
Christophe Leroybb297bb2020-04-01 21:11:54 -0700461static inline struct hstate *hstate_inode(struct inode *i)
462{
463 return NULL;
464}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465#endif /* !CONFIG_HUGETLBFS */
466
Adrian Bunkd2ba27e82007-05-06 14:49:00 -0700467#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
468unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
469 unsigned long len, unsigned long pgoff,
470 unsigned long flags);
471#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
472
Andi Kleena5516432008-07-23 21:27:41 -0700473#ifdef CONFIG_HUGETLB_PAGE
474
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700475#define HSTATE_NAME_LEN 32
Andi Kleena5516432008-07-23 21:27:41 -0700476/* Defines one hugetlb page size */
477struct hstate {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700478 int next_nid_to_alloc;
479 int next_nid_to_free;
Andi Kleena5516432008-07-23 21:27:41 -0700480 unsigned int order;
481 unsigned long mask;
482 unsigned long max_huge_pages;
483 unsigned long nr_huge_pages;
484 unsigned long free_huge_pages;
485 unsigned long resv_huge_pages;
486 unsigned long surplus_huge_pages;
487 unsigned long nr_overcommit_huge_pages;
Aneesh Kumar K.V0edaecf2012-07-31 16:42:07 -0700488 struct list_head hugepage_activelist;
Andi Kleena5516432008-07-23 21:27:41 -0700489 struct list_head hugepage_freelists[MAX_NUMNODES];
490 unsigned int nr_huge_pages_node[MAX_NUMNODES];
491 unsigned int free_huge_pages_node[MAX_NUMNODES];
492 unsigned int surplus_huge_pages_node[MAX_NUMNODES];
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -0700493#ifdef CONFIG_CGROUP_HUGETLB
494 /* cgroup control files */
Mina Almasrycdc2fcf2020-04-01 21:11:11 -0700495 struct cftype cgroup_files_dfl[7];
496 struct cftype cgroup_files_legacy[9];
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -0700497#endif
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700498 char name[HSTATE_NAME_LEN];
Andi Kleena5516432008-07-23 21:27:41 -0700499};
500
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700501struct huge_bootmem_page {
502 struct list_head list;
503 struct hstate *hstate;
504};
505
Mike Kravetz70c35472015-09-08 15:01:54 -0700506struct page *alloc_huge_page(struct vm_area_struct *vma,
507 unsigned long addr, int avoid_reserve);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900508struct page *alloc_huge_page_node(struct hstate *h, int nid);
Michal Hocko3e59fcb2017-07-10 15:49:11 -0700509struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
510 nodemask_t *nmask);
Michal Hocko389c8172018-01-31 16:21:03 -0800511struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
512 unsigned long address);
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -0800513struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
514 int nid, nodemask_t *nmask);
Mike Kravetzab76ad52015-09-08 15:01:50 -0700515int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
516 pgoff_t idx);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900517
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700518/* arch callback */
Aneesh Kumar K.Ve24a1302017-07-28 10:31:25 +0530519int __init __alloc_bootmem_huge_page(struct hstate *h);
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700520int __init alloc_bootmem_huge_page(struct hstate *h);
521
Vaishali Thakkar9fee0212016-05-19 17:11:04 -0700522void __init hugetlb_bad_size(void);
Andi Kleene5ff2152008-07-23 21:27:42 -0700523void __init hugetlb_add_hstate(unsigned order);
524struct hstate *size_to_hstate(unsigned long size);
525
526#ifndef HUGE_MAX_HSTATE
527#define HUGE_MAX_HSTATE 1
528#endif
529
530extern struct hstate hstates[HUGE_MAX_HSTATE];
531extern unsigned int default_hstate_idx;
532
533#define default_hstate (hstates[default_hstate_idx])
Andi Kleena5516432008-07-23 21:27:41 -0700534
Andi Kleena5516432008-07-23 21:27:41 -0700535static inline struct hstate *hstate_file(struct file *f)
536{
Al Viro496ad9a2013-01-23 17:07:38 -0500537 return hstate_inode(file_inode(f));
Andi Kleena5516432008-07-23 21:27:41 -0700538}
539
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700540static inline struct hstate *hstate_sizelog(int page_size_log)
541{
542 if (!page_size_log)
543 return &default_hstate;
Sasha Levin97ad2be2014-12-10 15:44:13 -0800544
545 return size_to_hstate(1UL << page_size_log);
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700546}
547
Andi Kleena137e1c2008-07-23 21:27:43 -0700548static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
Andi Kleena5516432008-07-23 21:27:41 -0700549{
Andi Kleena137e1c2008-07-23 21:27:43 -0700550 return hstate_file(vma->vm_file);
Andi Kleena5516432008-07-23 21:27:41 -0700551}
552
553static inline unsigned long huge_page_size(struct hstate *h)
554{
555 return (unsigned long)PAGE_SIZE << h->order;
556}
557
Mel Gorman08fba692009-01-06 14:38:53 -0800558extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
559
Mel Gorman33402892009-01-06 14:38:54 -0800560extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
561
Andi Kleena5516432008-07-23 21:27:41 -0700562static inline unsigned long huge_page_mask(struct hstate *h)
563{
564 return h->mask;
565}
566
567static inline unsigned int huge_page_order(struct hstate *h)
568{
569 return h->order;
570}
571
572static inline unsigned huge_page_shift(struct hstate *h)
573{
574 return h->order + PAGE_SHIFT;
575}
576
Luiz Capitulinobae7f4a2014-06-04 16:07:08 -0700577static inline bool hstate_is_gigantic(struct hstate *h)
578{
579 return huge_page_order(h) >= MAX_ORDER;
580}
581
Andi Kleena5516432008-07-23 21:27:41 -0700582static inline unsigned int pages_per_huge_page(struct hstate *h)
583{
584 return 1 << h->order;
585}
586
587static inline unsigned int blocks_per_huge_page(struct hstate *h)
588{
589 return huge_page_size(h) / 512;
590}
591
592#include <asm/hugetlb.h>
593
Chris Metcalfd9ed9fa2012-04-01 14:01:34 -0400594#ifndef arch_make_huge_pte
595static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
596 struct page *page, int writable)
597{
598 return entry;
599}
600#endif
601
Andi Kleene5ff2152008-07-23 21:27:42 -0700602static inline struct hstate *page_hstate(struct page *page)
603{
Sasha Levin309381fea2014-01-23 15:52:54 -0800604 VM_BUG_ON_PAGE(!PageHuge(page), page);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -0700605 return size_to_hstate(page_size(page));
Andi Kleene5ff2152008-07-23 21:27:42 -0700606}
607
Andi Kleenaa50d3a2010-10-06 21:45:00 +0200608static inline unsigned hstate_index_to_shift(unsigned index)
609{
610 return hstates[index].order + PAGE_SHIFT;
611}
612
Aneesh Kumar K.V972dc4d2012-07-31 16:42:00 -0700613static inline int hstate_index(struct hstate *h)
614{
615 return h - hstates;
616}
617
Zhang Yi13d60f42013-06-25 21:19:31 +0800618pgoff_t __basepage_index(struct page *page);
619
620/* Return page->index in PAGE_SIZE units */
621static inline pgoff_t basepage_index(struct page *page)
622{
623 if (!PageCompound(page))
624 return page->index;
625
626 return __basepage_index(page);
627}
628
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700629extern int dissolve_free_huge_page(struct page *page);
Gerald Schaefer082d5b62016-10-07 17:01:10 -0700630extern int dissolve_free_huge_pages(unsigned long start_pfn,
631 unsigned long end_pfn);
Anshuman Khanduale693de12019-03-05 15:43:51 -0800632
Naoya Horiguchic177c812014-06-04 16:05:35 -0700633#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
Anshuman Khanduale693de12019-03-05 15:43:51 -0800634#ifndef arch_hugetlb_migration_supported
635static inline bool arch_hugetlb_migration_supported(struct hstate *h)
636{
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700637 if ((huge_page_shift(h) == PMD_SHIFT) ||
Anshuman Khandual9b553bf2019-03-05 15:43:48 -0800638 (huge_page_shift(h) == PUD_SHIFT) ||
639 (huge_page_shift(h) == PGDIR_SHIFT))
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700640 return true;
641 else
642 return false;
Anshuman Khanduale693de12019-03-05 15:43:51 -0800643}
Naoya Horiguchic177c812014-06-04 16:05:35 -0700644#endif
Anshuman Khanduale693de12019-03-05 15:43:51 -0800645#else
646static inline bool arch_hugetlb_migration_supported(struct hstate *h)
647{
648 return false;
649}
650#endif
651
652static inline bool hugepage_migration_supported(struct hstate *h)
653{
654 return arch_hugetlb_migration_supported(h);
Naoya Horiguchi83467ef2013-09-11 14:22:11 -0700655}
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700656
Anshuman Khandual7ed2c312019-03-05 15:43:44 -0800657/*
658 * Movability check is different as compared to migration check.
659 * It determines whether or not a huge page should be placed on
660 * movable zone or not. Movability of any huge page should be
661 * required only if huge page size is supported for migration.
662 * There wont be any reason for the huge page to be movable if
663 * it is not migratable to start with. Also the size of the huge
664 * page should be large enough to be placed under a movable zone
665 * and still feasible enough to be migratable. Just the presence
666 * in movable zone does not make the migration feasible.
667 *
668 * So even though large huge page sizes like the gigantic ones
669 * are migratable they should not be movable because its not
670 * feasible to migrate them from movable zone.
671 */
672static inline bool hugepage_movable_supported(struct hstate *h)
673{
674 if (!hugepage_migration_supported(h))
675 return false;
676
677 if (hstate_is_gigantic(h))
678 return false;
679 return true;
680}
681
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800682static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
683 struct mm_struct *mm, pte_t *pte)
684{
685 if (huge_page_size(h) == PMD_SIZE)
686 return pmd_lockptr(mm, (pmd_t *) pte);
687 VM_BUG_ON(huge_page_size(h) == PAGE_SIZE);
688 return &mm->page_table_lock;
689}
690
Dominik Dingel2531c8c2015-07-17 16:23:37 -0700691#ifndef hugepages_supported
692/*
693 * Some platform decide whether they support huge pages at boot
694 * time. Some of them, such as powerpc, set HPAGE_SHIFT to 0
695 * when there is no such support
696 */
697#define hugepages_supported() (HPAGE_SHIFT != 0)
698#endif
Nishanth Aravamudan457c1b22014-05-06 12:50:00 -0700699
Naoya Horiguchi5d317b22015-11-05 18:47:14 -0800700void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm);
701
702static inline void hugetlb_count_add(long l, struct mm_struct *mm)
703{
704 atomic_long_add(l, &mm->hugetlb_usage);
705}
706
707static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
708{
709 atomic_long_sub(l, &mm->hugetlb_usage);
710}
Punit Agrawale5251fd2017-07-06 15:39:50 -0700711
712#ifndef set_huge_swap_pte_at
713static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
714 pte_t *ptep, pte_t pte, unsigned long sz)
715{
716 set_huge_pte_at(mm, addr, ptep, pte);
717}
718#endif
Aneesh Kumar K.V023bdd02019-03-05 15:46:37 -0800719
720#ifndef huge_ptep_modify_prot_start
721#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
722static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
723 unsigned long addr, pte_t *ptep)
724{
725 return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
726}
727#endif
728
729#ifndef huge_ptep_modify_prot_commit
730#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
731static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
732 unsigned long addr, pte_t *ptep,
733 pte_t old_pte, pte_t pte)
734{
735 set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
736}
737#endif
738
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700739#else /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700740struct hstate {};
Jason Gunthorpe442a5a92019-07-11 20:54:40 -0700741
742static inline struct page *alloc_huge_page(struct vm_area_struct *vma,
743 unsigned long addr,
744 int avoid_reserve)
745{
746 return NULL;
747}
748
749static inline struct page *alloc_huge_page_node(struct hstate *h, int nid)
750{
751 return NULL;
752}
753
754static inline struct page *
755alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask)
756{
757 return NULL;
758}
759
760static inline struct page *alloc_huge_page_vma(struct hstate *h,
761 struct vm_area_struct *vma,
762 unsigned long address)
763{
764 return NULL;
765}
766
767static inline int __alloc_bootmem_huge_page(struct hstate *h)
768{
769 return 0;
770}
771
772static inline struct hstate *hstate_file(struct file *f)
773{
774 return NULL;
775}
776
777static inline struct hstate *hstate_sizelog(int page_size_log)
778{
779 return NULL;
780}
781
782static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
783{
784 return NULL;
785}
786
Jason Gunthorpe442a5a92019-07-11 20:54:40 -0700787static inline struct hstate *page_hstate(struct page *page)
788{
789 return NULL;
790}
791
792static inline unsigned long huge_page_size(struct hstate *h)
793{
794 return PAGE_SIZE;
795}
796
797static inline unsigned long huge_page_mask(struct hstate *h)
798{
799 return PAGE_MASK;
800}
801
802static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
803{
804 return PAGE_SIZE;
805}
806
807static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
808{
809 return PAGE_SIZE;
810}
811
812static inline unsigned int huge_page_order(struct hstate *h)
813{
814 return 0;
815}
816
817static inline unsigned int huge_page_shift(struct hstate *h)
818{
819 return PAGE_SHIFT;
820}
821
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700822static inline bool hstate_is_gigantic(struct hstate *h)
823{
824 return false;
825}
826
Andrea Righi510a35d2008-07-26 15:22:27 -0700827static inline unsigned int pages_per_huge_page(struct hstate *h)
828{
829 return 1;
830}
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700831
832static inline unsigned hstate_index_to_shift(unsigned index)
833{
834 return 0;
835}
836
837static inline int hstate_index(struct hstate *h)
838{
839 return 0;
840}
Zhang Yi13d60f42013-06-25 21:19:31 +0800841
842static inline pgoff_t basepage_index(struct page *page)
843{
844 return page->index;
845}
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700846
847static inline int dissolve_free_huge_page(struct page *page)
848{
849 return 0;
850}
851
852static inline int dissolve_free_huge_pages(unsigned long start_pfn,
853 unsigned long end_pfn)
854{
855 return 0;
856}
857
858static inline bool hugepage_migration_supported(struct hstate *h)
859{
860 return false;
861}
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800862
Anshuman Khandual7ed2c312019-03-05 15:43:44 -0800863static inline bool hugepage_movable_supported(struct hstate *h)
864{
865 return false;
866}
867
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800868static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
869 struct mm_struct *mm, pte_t *pte)
870{
871 return &mm->page_table_lock;
872}
Naoya Horiguchi5d317b22015-11-05 18:47:14 -0800873
874static inline void hugetlb_report_usage(struct seq_file *f, struct mm_struct *m)
875{
876}
877
878static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
879{
880}
Punit Agrawale5251fd2017-07-06 15:39:50 -0700881
882static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
883 pte_t *ptep, pte_t pte, unsigned long sz)
884{
885}
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700886#endif /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700887
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800888static inline spinlock_t *huge_pte_lock(struct hstate *h,
889 struct mm_struct *mm, pte_t *pte)
890{
891 spinlock_t *ptl;
892
893 ptl = huge_pte_lockptr(h, mm, pte);
894 spin_lock(ptl);
895 return ptl;
896}
897
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898#endif /* _LINUX_HUGETLB_H */