blob: 219962c9517ebec8bd27d208bfea9c8f1af04a88 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002#ifndef _LINUX_HUGETLB_H
3#define _LINUX_HUGETLB_H
4
Linus Torvaldsbe93d8c2011-05-26 12:03:50 -07005#include <linux/mm_types.h>
Sasha Levin309381fea2014-01-23 15:52:54 -08006#include <linux/mmdebug.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +04007#include <linux/fs.h>
Naoya Horiguchi8edf3442010-05-28 09:29:15 +09008#include <linux/hugetlb_inline.h>
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -07009#include <linux/cgroup.h>
Joonsoo Kim9119a412014-04-03 14:47:25 -070010#include <linux/list.h>
11#include <linux/kref.h>
Dan Williams888cdbc2016-01-15 16:56:32 -080012#include <asm/pgtable.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040013
Andrew Mortone9ea0e22009-09-24 14:47:45 -070014struct ctl_table;
15struct user_struct;
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -070016struct mmu_gather;
Andrew Mortone9ea0e22009-09-24 14:47:45 -070017
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070018#ifndef is_hugepd
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070019typedef struct { unsigned long pd; } hugepd_t;
20#define is_hugepd(hugepd) (0)
21#define __hugepd(x) ((hugepd_t) { (x) })
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070022#endif
23
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#ifdef CONFIG_HUGETLB_PAGE
25
26#include <linux/mempolicy.h>
Adam Litke516dffd2007-03-01 15:46:08 -080027#include <linux/shm.h>
David Gibson63551ae2005-06-21 17:14:44 -070028#include <asm/tlbflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
David Gibson90481622012-03-21 16:34:12 -070030struct hugepage_subpool {
31 spinlock_t lock;
32 long count;
Mike Kravetzc6a91822015-04-15 16:13:36 -070033 long max_hpages; /* Maximum huge pages or -1 if no maximum. */
34 long used_hpages; /* Used count against maximum, includes */
35 /* both alloced and reserved pages. */
36 struct hstate *hstate;
37 long min_hpages; /* Minimum huge pages or -1 if no minimum. */
38 long rsv_hpages; /* Pages reserved against global pool to */
39 /* sasitfy minimum size. */
David Gibson90481622012-03-21 16:34:12 -070040};
41
Joonsoo Kim9119a412014-04-03 14:47:25 -070042struct resv_map {
43 struct kref refs;
Davidlohr Bueso7b24d862014-04-03 14:47:27 -070044 spinlock_t lock;
Joonsoo Kim9119a412014-04-03 14:47:25 -070045 struct list_head regions;
Mike Kravetz5e911372015-09-08 15:01:28 -070046 long adds_in_progress;
47 struct list_head region_cache;
48 long region_cache_count;
Mina Almasrye9fe92a2020-04-01 21:11:21 -070049#ifdef CONFIG_CGROUP_HUGETLB
50 /*
51 * On private mappings, the counter to uncharge reservations is stored
52 * here. If these fields are 0, then either the mapping is shared, or
53 * cgroup accounting is disabled for this resv_map.
54 */
55 struct page_counter *reservation_counter;
56 unsigned long pages_per_hpage;
57 struct cgroup_subsys_state *css;
58#endif
Joonsoo Kim9119a412014-04-03 14:47:25 -070059};
Mina Almasry075a61d2020-04-01 21:11:28 -070060
61/*
62 * Region tracking -- allows tracking of reservations and instantiated pages
63 * across the pages in a mapping.
64 *
65 * The region data structures are embedded into a resv_map and protected
66 * by a resv_map's lock. The set of regions within the resv_map represent
67 * reservations for huge pages, or huge pages that have already been
68 * instantiated within the map. The from and to elements are huge page
69 * indicies into the associated mapping. from indicates the starting index
70 * of the region. to represents the first index past the end of the region.
71 *
72 * For example, a file region structure with from == 0 and to == 4 represents
73 * four huge pages in a mapping. It is important to note that the to element
74 * represents the first element past the end of the region. This is used in
75 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
76 *
77 * Interval notation of the form [from, to) will be used to indicate that
78 * the endpoint from is inclusive and to is exclusive.
79 */
80struct file_region {
81 struct list_head link;
82 long from;
83 long to;
84#ifdef CONFIG_CGROUP_HUGETLB
85 /*
86 * On shared mappings, each reserved region appears as a struct
87 * file_region in resv_map. These fields hold the info needed to
88 * uncharge each reservation.
89 */
90 struct page_counter *reservation_counter;
91 struct cgroup_subsys_state *css;
92#endif
93};
94
Joonsoo Kim9119a412014-04-03 14:47:25 -070095extern struct resv_map *resv_map_alloc(void);
96void resv_map_release(struct kref *ref);
97
Aneesh Kumar K.Vc3f38a32012-07-31 16:42:10 -070098extern spinlock_t hugetlb_lock;
99extern int hugetlb_max_hstate __read_mostly;
100#define for_each_hstate(h) \
101 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
102
Mike Kravetz7ca02d0a2015-04-15 16:13:42 -0700103struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
104 long min_hpages);
David Gibson90481622012-03-21 16:34:12 -0700105void hugepage_put_subpool(struct hugepage_subpool *spool);
106
Mel Gormana1e78772008-07-23 21:27:23 -0700107void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
Alexey Dobriyan8d65af72009-09-23 15:57:19 -0700108int hugetlb_sysctl_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
109int hugetlb_overcommit_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
110int hugetlb_treat_movable_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
Lee Schermerhorn06808b02009-12-14 17:58:21 -0800111
112#ifdef CONFIG_NUMA
113int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int,
114 void __user *, size_t *, loff_t *);
115#endif
116
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
Michel Lespinasse28a35712013-02-22 16:35:55 -0800118long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
119 struct page **, struct vm_area_struct **,
Andrea Arcangeli87ffc112017-02-22 15:43:13 -0800120 unsigned long *, unsigned long *, long, unsigned int,
121 int *);
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700122void unmap_hugepage_range(struct vm_area_struct *,
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700123 unsigned long, unsigned long, struct page *);
Mel Gormand8333522012-07-31 16:46:20 -0700124void __unmap_hugepage_range_final(struct mmu_gather *tlb,
125 struct vm_area_struct *vma,
126 unsigned long start, unsigned long end,
127 struct page *ref_page);
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700128void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
129 unsigned long start, unsigned long end,
130 struct page *ref_page);
Alexey Dobriyane1759c22008-10-15 23:50:22 +0400131void hugetlb_report_meminfo(struct seq_file *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132int hugetlb_report_node_meminfo(int, char *);
David Rientjes949f7ec2013-04-29 15:07:48 -0700133void hugetlb_show_meminfo(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134unsigned long hugetlb_total_pages(void);
Souptick Joarder2b740302018-08-23 17:01:36 -0700135vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
Hugh Dickins788c7df2009-06-23 13:49:05 +0100136 unsigned long address, unsigned int flags);
Mike Kravetz8fb5deb2017-02-22 15:42:52 -0800137int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte,
138 struct vm_area_struct *dst_vma,
139 unsigned long dst_addr,
140 unsigned long src_addr,
141 struct page **pagep);
Mel Gormana1e78772008-07-23 21:27:23 -0700142int hugetlb_reserve_pages(struct inode *inode, long from, long to,
Mel Gorman5a6fe122009-02-10 14:02:27 +0000143 struct vm_area_struct *vma,
KOSAKI Motohiroca16d142011-05-26 19:16:19 +0900144 vm_flags_t vm_flags);
Mike Kravetzb5cec282015-09-08 15:01:41 -0700145long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
146 long freed);
Naoya Horiguchi31caf662013-09-11 14:21:59 -0700147bool isolate_huge_page(struct page *page, struct list_head *list);
148void putback_active_hugepage(struct page *page);
Michal Hockoab5ac902018-01-31 16:20:48 -0800149void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason);
Atsushi Kumagai8f1d26d2014-07-30 16:08:39 -0700150void free_huge_page(struct page *page);
zhong jiang72e29362016-10-07 17:02:01 -0700151void hugetlb_fix_reserve_counts(struct inode *inode);
Mike Kravetzc672c7f2015-09-08 15:01:35 -0700152extern struct mutex *hugetlb_fault_mutex_table;
Wei Yang188b04a2019-11-30 17:57:02 -0800153u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154
Steve Capper3212b532013-04-23 12:35:02 +0100155pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
Steve Capper3212b532013-04-23 12:35:02 +0100156
Mike Kravetzc0d03812020-04-01 21:11:05 -0700157struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage);
158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159extern int sysctl_hugetlb_shm_group;
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700160extern struct list_head huge_boot_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
David Gibson63551ae2005-06-21 17:14:44 -0700162/* arch callbacks */
163
Andi Kleena5516432008-07-23 21:27:41 -0700164pte_t *huge_pte_alloc(struct mm_struct *mm,
165 unsigned long addr, unsigned long sz);
Punit Agrawal7868a202017-07-06 15:39:42 -0700166pte_t *huge_pte_offset(struct mm_struct *mm,
167 unsigned long addr, unsigned long sz);
Chen, Kenneth W39dde652006-12-06 20:32:03 -0800168int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep);
Mike Kravetz017b1662018-10-05 15:51:29 -0700169void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
170 unsigned long *start, unsigned long *end);
David Gibson63551ae2005-06-21 17:14:44 -0700171struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
172 int write);
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700173struct page *follow_huge_pd(struct vm_area_struct *vma,
174 unsigned long address, hugepd_t hpd,
175 int flags, int pdshift);
David Gibson63551ae2005-06-21 17:14:44 -0700176struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
Naoya Horiguchie66f17f2015-02-11 15:25:22 -0800177 pmd_t *pmd, int flags);
Andi Kleenceb86872008-07-23 21:27:50 -0700178struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
Naoya Horiguchie66f17f2015-02-11 15:25:22 -0800179 pud_t *pud, int flags);
Anshuman Khandualfaaa5b62017-07-06 15:38:50 -0700180struct page *follow_huge_pgd(struct mm_struct *mm, unsigned long address,
181 pgd_t *pgd, int flags);
182
David Gibson63551ae2005-06-21 17:14:44 -0700183int pmd_huge(pmd_t pmd);
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300184int pud_huge(pud_t pud);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100185unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800186 unsigned long address, unsigned long end, pgprot_t newprot);
David Gibson63551ae2005-06-21 17:14:44 -0700187
Aneesh Kumar K.Vd5ed7442017-07-06 15:38:47 -0700188bool is_hugetlb_entry_migration(pte_t pte);
Michal Hockoab5ac902018-01-31 16:20:48 -0800189
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190#else /* !CONFIG_HUGETLB_PAGE */
191
Mel Gormana1e78772008-07-23 21:27:23 -0700192static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
193{
194}
195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196static inline unsigned long hugetlb_total_pages(void)
197{
198 return 0;
199}
200
Mike Kravetzc0d03812020-04-01 21:11:05 -0700201static inline struct address_space *hugetlb_page_mapping_lock_write(
202 struct page *hpage)
203{
204 return NULL;
205}
206
Mike Kravetz017b1662018-10-05 15:51:29 -0700207static inline int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr,
208 pte_t *ptep)
209{
210 return 0;
211}
212
213static inline void adjust_range_if_pmd_sharing_possible(
214 struct vm_area_struct *vma,
215 unsigned long *start, unsigned long *end)
216{
217}
218
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800219static inline long follow_hugetlb_page(struct mm_struct *mm,
220 struct vm_area_struct *vma, struct page **pages,
221 struct vm_area_struct **vmas, unsigned long *position,
222 unsigned long *nr_pages, long i, unsigned int flags,
223 int *nonblocking)
224{
225 BUG();
226 return 0;
227}
228
229static inline struct page *follow_huge_addr(struct mm_struct *mm,
230 unsigned long address, int write)
231{
232 return ERR_PTR(-EINVAL);
233}
234
235static inline int copy_hugetlb_page_range(struct mm_struct *dst,
236 struct mm_struct *src, struct vm_area_struct *vma)
237{
238 BUG();
239 return 0;
240}
241
Alexey Dobriyane1759c22008-10-15 23:50:22 +0400242static inline void hugetlb_report_meminfo(struct seq_file *m)
243{
244}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800245
246static inline int hugetlb_report_node_meminfo(int nid, char *buf)
247{
248 return 0;
249}
250
David Rientjes949f7ec2013-04-29 15:07:48 -0700251static inline void hugetlb_show_meminfo(void)
252{
253}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800254
255static inline struct page *follow_huge_pd(struct vm_area_struct *vma,
256 unsigned long address, hugepd_t hpd, int flags,
257 int pdshift)
258{
259 return NULL;
260}
261
262static inline struct page *follow_huge_pmd(struct mm_struct *mm,
263 unsigned long address, pmd_t *pmd, int flags)
264{
265 return NULL;
266}
267
268static inline struct page *follow_huge_pud(struct mm_struct *mm,
269 unsigned long address, pud_t *pud, int flags)
270{
271 return NULL;
272}
273
274static inline struct page *follow_huge_pgd(struct mm_struct *mm,
275 unsigned long address, pgd_t *pgd, int flags)
276{
277 return NULL;
278}
279
280static inline int prepare_hugepage_range(struct file *file,
281 unsigned long addr, unsigned long len)
282{
283 return -EINVAL;
284}
285
286static inline int pmd_huge(pmd_t pmd)
287{
288 return 0;
289}
290
291static inline int pud_huge(pud_t pud)
292{
293 return 0;
294}
295
296static inline int is_hugepage_only_range(struct mm_struct *mm,
297 unsigned long addr, unsigned long len)
298{
299 return 0;
300}
301
302static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
303 unsigned long addr, unsigned long end,
304 unsigned long floor, unsigned long ceiling)
305{
306 BUG();
307}
308
309static inline int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
310 pte_t *dst_pte,
311 struct vm_area_struct *dst_vma,
312 unsigned long dst_addr,
313 unsigned long src_addr,
314 struct page **pagep)
315{
316 BUG();
317 return 0;
318}
319
320static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
321 unsigned long sz)
322{
323 return NULL;
324}
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700325
Naoya Horiguchif40386a2013-12-12 17:12:19 -0800326static inline bool isolate_huge_page(struct page *page, struct list_head *list)
327{
328 return false;
329}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800331static inline void putback_active_hugepage(struct page *page)
332{
333}
334
335static inline void move_hugetlb_state(struct page *oldpage,
336 struct page *newpage, int reason)
337{
338}
339
340static inline unsigned long hugetlb_change_protection(
341 struct vm_area_struct *vma, unsigned long address,
342 unsigned long end, pgprot_t newprot)
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100343{
344 return 0;
345}
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800346
Mel Gormand8333522012-07-31 16:46:20 -0700347static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
348 struct vm_area_struct *vma, unsigned long start,
349 unsigned long end, struct page *ref_page)
350{
351 BUG();
352}
353
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700354static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
355 struct vm_area_struct *vma, unsigned long start,
356 unsigned long end, struct page *ref_page)
357{
358 BUG();
359}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800360
Souptick Joardera953e772019-03-28 20:43:51 -0700361static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800362 struct vm_area_struct *vma, unsigned long address,
363 unsigned int flags)
Souptick Joardera953e772019-03-28 20:43:51 -0700364{
365 BUG();
366 return 0;
367}
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700368
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369#endif /* !CONFIG_HUGETLB_PAGE */
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +0530370/*
371 * hugepages at page global directory. If arch support
372 * hugepages at pgd level, they need to define this.
373 */
374#ifndef pgd_huge
375#define pgd_huge(x) 0
376#endif
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300377#ifndef p4d_huge
378#define p4d_huge(x) 0
379#endif
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +0530380
381#ifndef pgd_write
382static inline int pgd_write(pgd_t pgd)
383{
384 BUG();
385 return 0;
386}
387#endif
388
Eric B Munson4e527802009-09-21 17:03:47 -0700389#define HUGETLB_ANON_FILE "anon_hugepage"
390
Eric B Munson6bfde052009-09-21 17:03:43 -0700391enum {
392 /*
393 * The file will be used as an shm file so shmfs accounting rules
394 * apply
395 */
396 HUGETLB_SHMFS_INODE = 1,
Eric B Munson4e527802009-09-21 17:03:47 -0700397 /*
398 * The file is being created on the internal vfs mount and shmfs
399 * accounting rules do not apply
400 */
401 HUGETLB_ANONHUGE_INODE = 2,
Eric B Munson6bfde052009-09-21 17:03:43 -0700402};
403
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404#ifdef CONFIG_HUGETLBFS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405struct hugetlbfs_sb_info {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 long max_inodes; /* inodes allowed */
407 long free_inodes; /* inodes free */
408 spinlock_t stat_lock;
Andi Kleena137e1c2008-07-23 21:27:43 -0700409 struct hstate *hstate;
David Gibson90481622012-03-21 16:34:12 -0700410 struct hugepage_subpool *spool;
David Howells4a252202017-07-05 16:24:18 +0100411 kuid_t uid;
412 kgid_t gid;
413 umode_t mode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414};
415
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
417{
418 return sb->s_fs_info;
419}
420
Marc-André Lureauda14c1e2018-01-31 16:19:22 -0800421struct hugetlbfs_inode_info {
422 struct shared_policy policy;
423 struct inode vfs_inode;
Marc-André Lureauff62a342018-01-31 16:19:25 -0800424 unsigned int seals;
Marc-André Lureauda14c1e2018-01-31 16:19:22 -0800425};
426
427static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
428{
429 return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
430}
431
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -0800432extern const struct file_operations hugetlbfs_file_operations;
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +0400433extern const struct vm_operations_struct hugetlb_vm_ops;
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700434struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
Andi Kleen42d73952012-12-11 16:01:34 -0800435 struct user_struct **user, int creat_flags,
436 int page_size_log);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437
Yaowei Bai719ff3212016-01-14 15:18:51 -0800438static inline bool is_file_hugepages(struct file *file)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439{
Adam Litke516dffd2007-03-01 15:46:08 -0800440 if (file->f_op == &hugetlbfs_file_operations)
Yaowei Bai719ff3212016-01-14 15:18:51 -0800441 return true;
Adam Litke516dffd2007-03-01 15:46:08 -0800442
Yaowei Bai719ff3212016-01-14 15:18:51 -0800443 return is_file_shm_hugepages(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444}
445
Andi Kleen42d73952012-12-11 16:01:34 -0800446
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447#else /* !CONFIG_HUGETLBFS */
448
Yaowei Bai719ff3212016-01-14 15:18:51 -0800449#define is_file_hugepages(file) false
Steven Truelove40716e22012-03-21 16:34:14 -0700450static inline struct file *
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700451hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
452 struct user_struct **user, int creat_flags,
Andi Kleen42d73952012-12-11 16:01:34 -0800453 int page_size_log)
Andrew Mortone9ea0e22009-09-24 14:47:45 -0700454{
455 return ERR_PTR(-ENOSYS);
456}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457
458#endif /* !CONFIG_HUGETLBFS */
459
Adrian Bunkd2ba27e82007-05-06 14:49:00 -0700460#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
461unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
462 unsigned long len, unsigned long pgoff,
463 unsigned long flags);
464#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
465
Andi Kleena5516432008-07-23 21:27:41 -0700466#ifdef CONFIG_HUGETLB_PAGE
467
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700468#define HSTATE_NAME_LEN 32
Andi Kleena5516432008-07-23 21:27:41 -0700469/* Defines one hugetlb page size */
470struct hstate {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700471 int next_nid_to_alloc;
472 int next_nid_to_free;
Andi Kleena5516432008-07-23 21:27:41 -0700473 unsigned int order;
474 unsigned long mask;
475 unsigned long max_huge_pages;
476 unsigned long nr_huge_pages;
477 unsigned long free_huge_pages;
478 unsigned long resv_huge_pages;
479 unsigned long surplus_huge_pages;
480 unsigned long nr_overcommit_huge_pages;
Aneesh Kumar K.V0edaecf2012-07-31 16:42:07 -0700481 struct list_head hugepage_activelist;
Andi Kleena5516432008-07-23 21:27:41 -0700482 struct list_head hugepage_freelists[MAX_NUMNODES];
483 unsigned int nr_huge_pages_node[MAX_NUMNODES];
484 unsigned int free_huge_pages_node[MAX_NUMNODES];
485 unsigned int surplus_huge_pages_node[MAX_NUMNODES];
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -0700486#ifdef CONFIG_CGROUP_HUGETLB
487 /* cgroup control files */
Mina Almasrycdc2fcf2020-04-01 21:11:11 -0700488 struct cftype cgroup_files_dfl[7];
489 struct cftype cgroup_files_legacy[9];
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -0700490#endif
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700491 char name[HSTATE_NAME_LEN];
Andi Kleena5516432008-07-23 21:27:41 -0700492};
493
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700494struct huge_bootmem_page {
495 struct list_head list;
496 struct hstate *hstate;
497};
498
Mike Kravetz70c35472015-09-08 15:01:54 -0700499struct page *alloc_huge_page(struct vm_area_struct *vma,
500 unsigned long addr, int avoid_reserve);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900501struct page *alloc_huge_page_node(struct hstate *h, int nid);
Michal Hocko3e59fcb2017-07-10 15:49:11 -0700502struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
503 nodemask_t *nmask);
Michal Hocko389c8172018-01-31 16:21:03 -0800504struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
505 unsigned long address);
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -0800506struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
507 int nid, nodemask_t *nmask);
Mike Kravetzab76ad52015-09-08 15:01:50 -0700508int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
509 pgoff_t idx);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900510
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700511/* arch callback */
Aneesh Kumar K.Ve24a1302017-07-28 10:31:25 +0530512int __init __alloc_bootmem_huge_page(struct hstate *h);
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700513int __init alloc_bootmem_huge_page(struct hstate *h);
514
Vaishali Thakkar9fee0212016-05-19 17:11:04 -0700515void __init hugetlb_bad_size(void);
Andi Kleene5ff2152008-07-23 21:27:42 -0700516void __init hugetlb_add_hstate(unsigned order);
517struct hstate *size_to_hstate(unsigned long size);
518
519#ifndef HUGE_MAX_HSTATE
520#define HUGE_MAX_HSTATE 1
521#endif
522
523extern struct hstate hstates[HUGE_MAX_HSTATE];
524extern unsigned int default_hstate_idx;
525
526#define default_hstate (hstates[default_hstate_idx])
Andi Kleena5516432008-07-23 21:27:41 -0700527
Andi Kleena137e1c2008-07-23 21:27:43 -0700528static inline struct hstate *hstate_inode(struct inode *i)
Andi Kleena5516432008-07-23 21:27:41 -0700529{
Chen Gang7fab3582016-05-20 16:57:59 -0700530 return HUGETLBFS_SB(i->i_sb)->hstate;
Andi Kleena5516432008-07-23 21:27:41 -0700531}
532
533static inline struct hstate *hstate_file(struct file *f)
534{
Al Viro496ad9a2013-01-23 17:07:38 -0500535 return hstate_inode(file_inode(f));
Andi Kleena5516432008-07-23 21:27:41 -0700536}
537
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700538static inline struct hstate *hstate_sizelog(int page_size_log)
539{
540 if (!page_size_log)
541 return &default_hstate;
Sasha Levin97ad2be2014-12-10 15:44:13 -0800542
543 return size_to_hstate(1UL << page_size_log);
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700544}
545
Andi Kleena137e1c2008-07-23 21:27:43 -0700546static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
Andi Kleena5516432008-07-23 21:27:41 -0700547{
Andi Kleena137e1c2008-07-23 21:27:43 -0700548 return hstate_file(vma->vm_file);
Andi Kleena5516432008-07-23 21:27:41 -0700549}
550
551static inline unsigned long huge_page_size(struct hstate *h)
552{
553 return (unsigned long)PAGE_SIZE << h->order;
554}
555
Mel Gorman08fba692009-01-06 14:38:53 -0800556extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
557
Mel Gorman33402892009-01-06 14:38:54 -0800558extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
559
Andi Kleena5516432008-07-23 21:27:41 -0700560static inline unsigned long huge_page_mask(struct hstate *h)
561{
562 return h->mask;
563}
564
565static inline unsigned int huge_page_order(struct hstate *h)
566{
567 return h->order;
568}
569
570static inline unsigned huge_page_shift(struct hstate *h)
571{
572 return h->order + PAGE_SHIFT;
573}
574
Luiz Capitulinobae7f4a2014-06-04 16:07:08 -0700575static inline bool hstate_is_gigantic(struct hstate *h)
576{
577 return huge_page_order(h) >= MAX_ORDER;
578}
579
Andi Kleena5516432008-07-23 21:27:41 -0700580static inline unsigned int pages_per_huge_page(struct hstate *h)
581{
582 return 1 << h->order;
583}
584
585static inline unsigned int blocks_per_huge_page(struct hstate *h)
586{
587 return huge_page_size(h) / 512;
588}
589
590#include <asm/hugetlb.h>
591
Chris Metcalfd9ed9fa2012-04-01 14:01:34 -0400592#ifndef arch_make_huge_pte
593static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
594 struct page *page, int writable)
595{
596 return entry;
597}
598#endif
599
Andi Kleene5ff2152008-07-23 21:27:42 -0700600static inline struct hstate *page_hstate(struct page *page)
601{
Sasha Levin309381fea2014-01-23 15:52:54 -0800602 VM_BUG_ON_PAGE(!PageHuge(page), page);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -0700603 return size_to_hstate(page_size(page));
Andi Kleene5ff2152008-07-23 21:27:42 -0700604}
605
Andi Kleenaa50d3a2010-10-06 21:45:00 +0200606static inline unsigned hstate_index_to_shift(unsigned index)
607{
608 return hstates[index].order + PAGE_SHIFT;
609}
610
Aneesh Kumar K.V972dc4d2012-07-31 16:42:00 -0700611static inline int hstate_index(struct hstate *h)
612{
613 return h - hstates;
614}
615
Zhang Yi13d60f42013-06-25 21:19:31 +0800616pgoff_t __basepage_index(struct page *page);
617
618/* Return page->index in PAGE_SIZE units */
619static inline pgoff_t basepage_index(struct page *page)
620{
621 if (!PageCompound(page))
622 return page->index;
623
624 return __basepage_index(page);
625}
626
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700627extern int dissolve_free_huge_page(struct page *page);
Gerald Schaefer082d5b62016-10-07 17:01:10 -0700628extern int dissolve_free_huge_pages(unsigned long start_pfn,
629 unsigned long end_pfn);
Anshuman Khanduale693de12019-03-05 15:43:51 -0800630
Naoya Horiguchic177c812014-06-04 16:05:35 -0700631#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
Anshuman Khanduale693de12019-03-05 15:43:51 -0800632#ifndef arch_hugetlb_migration_supported
633static inline bool arch_hugetlb_migration_supported(struct hstate *h)
634{
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700635 if ((huge_page_shift(h) == PMD_SHIFT) ||
Anshuman Khandual9b553bf2019-03-05 15:43:48 -0800636 (huge_page_shift(h) == PUD_SHIFT) ||
637 (huge_page_shift(h) == PGDIR_SHIFT))
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700638 return true;
639 else
640 return false;
Anshuman Khanduale693de12019-03-05 15:43:51 -0800641}
Naoya Horiguchic177c812014-06-04 16:05:35 -0700642#endif
Anshuman Khanduale693de12019-03-05 15:43:51 -0800643#else
644static inline bool arch_hugetlb_migration_supported(struct hstate *h)
645{
646 return false;
647}
648#endif
649
650static inline bool hugepage_migration_supported(struct hstate *h)
651{
652 return arch_hugetlb_migration_supported(h);
Naoya Horiguchi83467ef2013-09-11 14:22:11 -0700653}
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700654
Anshuman Khandual7ed2c312019-03-05 15:43:44 -0800655/*
656 * Movability check is different as compared to migration check.
657 * It determines whether or not a huge page should be placed on
658 * movable zone or not. Movability of any huge page should be
659 * required only if huge page size is supported for migration.
660 * There wont be any reason for the huge page to be movable if
661 * it is not migratable to start with. Also the size of the huge
662 * page should be large enough to be placed under a movable zone
663 * and still feasible enough to be migratable. Just the presence
664 * in movable zone does not make the migration feasible.
665 *
666 * So even though large huge page sizes like the gigantic ones
667 * are migratable they should not be movable because its not
668 * feasible to migrate them from movable zone.
669 */
670static inline bool hugepage_movable_supported(struct hstate *h)
671{
672 if (!hugepage_migration_supported(h))
673 return false;
674
675 if (hstate_is_gigantic(h))
676 return false;
677 return true;
678}
679
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800680static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
681 struct mm_struct *mm, pte_t *pte)
682{
683 if (huge_page_size(h) == PMD_SIZE)
684 return pmd_lockptr(mm, (pmd_t *) pte);
685 VM_BUG_ON(huge_page_size(h) == PAGE_SIZE);
686 return &mm->page_table_lock;
687}
688
Dominik Dingel2531c8c2015-07-17 16:23:37 -0700689#ifndef hugepages_supported
690/*
691 * Some platform decide whether they support huge pages at boot
692 * time. Some of them, such as powerpc, set HPAGE_SHIFT to 0
693 * when there is no such support
694 */
695#define hugepages_supported() (HPAGE_SHIFT != 0)
696#endif
Nishanth Aravamudan457c1b22014-05-06 12:50:00 -0700697
Naoya Horiguchi5d317b22015-11-05 18:47:14 -0800698void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm);
699
700static inline void hugetlb_count_add(long l, struct mm_struct *mm)
701{
702 atomic_long_add(l, &mm->hugetlb_usage);
703}
704
705static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
706{
707 atomic_long_sub(l, &mm->hugetlb_usage);
708}
Punit Agrawale5251fd2017-07-06 15:39:50 -0700709
710#ifndef set_huge_swap_pte_at
711static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
712 pte_t *ptep, pte_t pte, unsigned long sz)
713{
714 set_huge_pte_at(mm, addr, ptep, pte);
715}
716#endif
Aneesh Kumar K.V023bdd02019-03-05 15:46:37 -0800717
718#ifndef huge_ptep_modify_prot_start
719#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
720static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
721 unsigned long addr, pte_t *ptep)
722{
723 return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
724}
725#endif
726
727#ifndef huge_ptep_modify_prot_commit
728#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
729static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
730 unsigned long addr, pte_t *ptep,
731 pte_t old_pte, pte_t pte)
732{
733 set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
734}
735#endif
736
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700737#else /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700738struct hstate {};
Jason Gunthorpe442a5a92019-07-11 20:54:40 -0700739
740static inline struct page *alloc_huge_page(struct vm_area_struct *vma,
741 unsigned long addr,
742 int avoid_reserve)
743{
744 return NULL;
745}
746
747static inline struct page *alloc_huge_page_node(struct hstate *h, int nid)
748{
749 return NULL;
750}
751
752static inline struct page *
753alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask)
754{
755 return NULL;
756}
757
758static inline struct page *alloc_huge_page_vma(struct hstate *h,
759 struct vm_area_struct *vma,
760 unsigned long address)
761{
762 return NULL;
763}
764
765static inline int __alloc_bootmem_huge_page(struct hstate *h)
766{
767 return 0;
768}
769
770static inline struct hstate *hstate_file(struct file *f)
771{
772 return NULL;
773}
774
775static inline struct hstate *hstate_sizelog(int page_size_log)
776{
777 return NULL;
778}
779
780static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
781{
782 return NULL;
783}
784
785static inline struct hstate *hstate_inode(struct inode *i)
786{
787 return NULL;
788}
789
790static inline struct hstate *page_hstate(struct page *page)
791{
792 return NULL;
793}
794
795static inline unsigned long huge_page_size(struct hstate *h)
796{
797 return PAGE_SIZE;
798}
799
800static inline unsigned long huge_page_mask(struct hstate *h)
801{
802 return PAGE_MASK;
803}
804
805static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
806{
807 return PAGE_SIZE;
808}
809
810static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
811{
812 return PAGE_SIZE;
813}
814
815static inline unsigned int huge_page_order(struct hstate *h)
816{
817 return 0;
818}
819
820static inline unsigned int huge_page_shift(struct hstate *h)
821{
822 return PAGE_SHIFT;
823}
824
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700825static inline bool hstate_is_gigantic(struct hstate *h)
826{
827 return false;
828}
829
Andrea Righi510a35d2008-07-26 15:22:27 -0700830static inline unsigned int pages_per_huge_page(struct hstate *h)
831{
832 return 1;
833}
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700834
835static inline unsigned hstate_index_to_shift(unsigned index)
836{
837 return 0;
838}
839
840static inline int hstate_index(struct hstate *h)
841{
842 return 0;
843}
Zhang Yi13d60f42013-06-25 21:19:31 +0800844
845static inline pgoff_t basepage_index(struct page *page)
846{
847 return page->index;
848}
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700849
850static inline int dissolve_free_huge_page(struct page *page)
851{
852 return 0;
853}
854
855static inline int dissolve_free_huge_pages(unsigned long start_pfn,
856 unsigned long end_pfn)
857{
858 return 0;
859}
860
861static inline bool hugepage_migration_supported(struct hstate *h)
862{
863 return false;
864}
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800865
Anshuman Khandual7ed2c312019-03-05 15:43:44 -0800866static inline bool hugepage_movable_supported(struct hstate *h)
867{
868 return false;
869}
870
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800871static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
872 struct mm_struct *mm, pte_t *pte)
873{
874 return &mm->page_table_lock;
875}
Naoya Horiguchi5d317b22015-11-05 18:47:14 -0800876
877static inline void hugetlb_report_usage(struct seq_file *f, struct mm_struct *m)
878{
879}
880
881static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
882{
883}
Punit Agrawale5251fd2017-07-06 15:39:50 -0700884
885static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
886 pte_t *ptep, pte_t pte, unsigned long sz)
887{
888}
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700889#endif /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700890
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800891static inline spinlock_t *huge_pte_lock(struct hstate *h,
892 struct mm_struct *mm, pte_t *pte)
893{
894 spinlock_t *ptl;
895
896 ptl = huge_pte_lockptr(h, mm, pte);
897 spin_lock(ptl);
898 return ptl;
899}
900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901#endif /* _LINUX_HUGETLB_H */