blob: a520bf26e5d8e0e3e0c90efd484d424a2cfe739b [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002#ifndef _LINUX_HUGETLB_H
3#define _LINUX_HUGETLB_H
4
Linus Torvaldsbe93d8c2011-05-26 12:03:50 -07005#include <linux/mm_types.h>
Sasha Levin309381fea2014-01-23 15:52:54 -08006#include <linux/mmdebug.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +04007#include <linux/fs.h>
Naoya Horiguchi8edf3442010-05-28 09:29:15 +09008#include <linux/hugetlb_inline.h>
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -07009#include <linux/cgroup.h>
Joonsoo Kim9119a412014-04-03 14:47:25 -070010#include <linux/list.h>
11#include <linux/kref.h>
Mike Rapoportca5999f2020-06-08 21:32:38 -070012#include <linux/pgtable.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040013
Andrew Mortone9ea0e22009-09-24 14:47:45 -070014struct ctl_table;
15struct user_struct;
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -070016struct mmu_gather;
Andrew Mortone9ea0e22009-09-24 14:47:45 -070017
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070018#ifndef is_hugepd
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070019typedef struct { unsigned long pd; } hugepd_t;
20#define is_hugepd(hugepd) (0)
21#define __hugepd(x) ((hugepd_t) { (x) })
Aneesh Kumar K.Ve2299292017-07-06 15:38:53 -070022#endif
23
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#ifdef CONFIG_HUGETLB_PAGE
25
26#include <linux/mempolicy.h>
Adam Litke516dffd2007-03-01 15:46:08 -080027#include <linux/shm.h>
David Gibson63551ae2005-06-21 17:14:44 -070028#include <asm/tlbflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
David Gibson90481622012-03-21 16:34:12 -070030struct hugepage_subpool {
31 spinlock_t lock;
32 long count;
Mike Kravetzc6a91822015-04-15 16:13:36 -070033 long max_hpages; /* Maximum huge pages or -1 if no maximum. */
34 long used_hpages; /* Used count against maximum, includes */
35 /* both alloced and reserved pages. */
36 struct hstate *hstate;
37 long min_hpages; /* Minimum huge pages or -1 if no minimum. */
38 long rsv_hpages; /* Pages reserved against global pool to */
39 /* sasitfy minimum size. */
David Gibson90481622012-03-21 16:34:12 -070040};
41
Joonsoo Kim9119a412014-04-03 14:47:25 -070042struct resv_map {
43 struct kref refs;
Davidlohr Bueso7b24d862014-04-03 14:47:27 -070044 spinlock_t lock;
Joonsoo Kim9119a412014-04-03 14:47:25 -070045 struct list_head regions;
Mike Kravetz5e911372015-09-08 15:01:28 -070046 long adds_in_progress;
47 struct list_head region_cache;
48 long region_cache_count;
Mina Almasrye9fe92a2020-04-01 21:11:21 -070049#ifdef CONFIG_CGROUP_HUGETLB
50 /*
51 * On private mappings, the counter to uncharge reservations is stored
52 * here. If these fields are 0, then either the mapping is shared, or
53 * cgroup accounting is disabled for this resv_map.
54 */
55 struct page_counter *reservation_counter;
56 unsigned long pages_per_hpage;
57 struct cgroup_subsys_state *css;
58#endif
Joonsoo Kim9119a412014-04-03 14:47:25 -070059};
Mina Almasry075a61d2020-04-01 21:11:28 -070060
61/*
62 * Region tracking -- allows tracking of reservations and instantiated pages
63 * across the pages in a mapping.
64 *
65 * The region data structures are embedded into a resv_map and protected
66 * by a resv_map's lock. The set of regions within the resv_map represent
67 * reservations for huge pages, or huge pages that have already been
68 * instantiated within the map. The from and to elements are huge page
69 * indicies into the associated mapping. from indicates the starting index
70 * of the region. to represents the first index past the end of the region.
71 *
72 * For example, a file region structure with from == 0 and to == 4 represents
73 * four huge pages in a mapping. It is important to note that the to element
74 * represents the first element past the end of the region. This is used in
75 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
76 *
77 * Interval notation of the form [from, to) will be used to indicate that
78 * the endpoint from is inclusive and to is exclusive.
79 */
80struct file_region {
81 struct list_head link;
82 long from;
83 long to;
84#ifdef CONFIG_CGROUP_HUGETLB
85 /*
86 * On shared mappings, each reserved region appears as a struct
87 * file_region in resv_map. These fields hold the info needed to
88 * uncharge each reservation.
89 */
90 struct page_counter *reservation_counter;
91 struct cgroup_subsys_state *css;
92#endif
93};
94
Joonsoo Kim9119a412014-04-03 14:47:25 -070095extern struct resv_map *resv_map_alloc(void);
96void resv_map_release(struct kref *ref);
97
Aneesh Kumar K.Vc3f38a32012-07-31 16:42:10 -070098extern spinlock_t hugetlb_lock;
99extern int hugetlb_max_hstate __read_mostly;
100#define for_each_hstate(h) \
101 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
102
Mike Kravetz7ca02d0a2015-04-15 16:13:42 -0700103struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
104 long min_hpages);
David Gibson90481622012-03-21 16:34:12 -0700105void hugepage_put_subpool(struct hugepage_subpool *spool);
106
Mel Gormana1e78772008-07-23 21:27:23 -0700107void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
Christoph Hellwig32927392020-04-24 08:43:38 +0200108int hugetlb_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *);
109int hugetlb_overcommit_handler(struct ctl_table *, int, void *, size_t *,
110 loff_t *);
111int hugetlb_treat_movable_handler(struct ctl_table *, int, void *, size_t *,
112 loff_t *);
113int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int, void *, size_t *,
114 loff_t *);
Lee Schermerhorn06808b02009-12-14 17:58:21 -0800115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
Michel Lespinasse28a35712013-02-22 16:35:55 -0800117long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
118 struct page **, struct vm_area_struct **,
Andrea Arcangeli87ffc112017-02-22 15:43:13 -0800119 unsigned long *, unsigned long *, long, unsigned int,
120 int *);
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700121void unmap_hugepage_range(struct vm_area_struct *,
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700122 unsigned long, unsigned long, struct page *);
Mel Gormand8333522012-07-31 16:46:20 -0700123void __unmap_hugepage_range_final(struct mmu_gather *tlb,
124 struct vm_area_struct *vma,
125 unsigned long start, unsigned long end,
126 struct page *ref_page);
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700127void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
128 unsigned long start, unsigned long end,
129 struct page *ref_page);
Alexey Dobriyane1759c22008-10-15 23:50:22 +0400130void hugetlb_report_meminfo(struct seq_file *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131int hugetlb_report_node_meminfo(int, char *);
David Rientjes949f7ec2013-04-29 15:07:48 -0700132void hugetlb_show_meminfo(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133unsigned long hugetlb_total_pages(void);
Souptick Joarder2b740302018-08-23 17:01:36 -0700134vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
Hugh Dickins788c7df2009-06-23 13:49:05 +0100135 unsigned long address, unsigned int flags);
Mike Kravetz8fb5deb2017-02-22 15:42:52 -0800136int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte,
137 struct vm_area_struct *dst_vma,
138 unsigned long dst_addr,
139 unsigned long src_addr,
140 struct page **pagep);
Mel Gormana1e78772008-07-23 21:27:23 -0700141int hugetlb_reserve_pages(struct inode *inode, long from, long to,
Mel Gorman5a6fe122009-02-10 14:02:27 +0000142 struct vm_area_struct *vma,
KOSAKI Motohiroca16d142011-05-26 19:16:19 +0900143 vm_flags_t vm_flags);
Mike Kravetzb5cec282015-09-08 15:01:41 -0700144long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
145 long freed);
Naoya Horiguchi31caf662013-09-11 14:21:59 -0700146bool isolate_huge_page(struct page *page, struct list_head *list);
147void putback_active_hugepage(struct page *page);
Michal Hockoab5ac902018-01-31 16:20:48 -0800148void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason);
Atsushi Kumagai8f1d26d2014-07-30 16:08:39 -0700149void free_huge_page(struct page *page);
zhong jiang72e29362016-10-07 17:02:01 -0700150void hugetlb_fix_reserve_counts(struct inode *inode);
Mike Kravetzc672c7f2015-09-08 15:01:35 -0700151extern struct mutex *hugetlb_fault_mutex_table;
Wei Yang188b04a2019-11-30 17:57:02 -0800152u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153
Steve Capper3212b532013-04-23 12:35:02 +0100154pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
Steve Capper3212b532013-04-23 12:35:02 +0100155
Mike Kravetzc0d03812020-04-01 21:11:05 -0700156struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage);
157
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158extern int sysctl_hugetlb_shm_group;
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700159extern struct list_head huge_boot_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160
David Gibson63551ae2005-06-21 17:14:44 -0700161/* arch callbacks */
162
Andi Kleena5516432008-07-23 21:27:41 -0700163pte_t *huge_pte_alloc(struct mm_struct *mm,
164 unsigned long addr, unsigned long sz);
Punit Agrawal7868a202017-07-06 15:39:42 -0700165pte_t *huge_pte_offset(struct mm_struct *mm,
166 unsigned long addr, unsigned long sz);
Mike Kravetz34ae2042020-08-11 18:31:38 -0700167int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
168 unsigned long *addr, pte_t *ptep);
Mike Kravetz017b1662018-10-05 15:51:29 -0700169void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
170 unsigned long *start, unsigned long *end);
David Gibson63551ae2005-06-21 17:14:44 -0700171struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
172 int write);
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700173struct page *follow_huge_pd(struct vm_area_struct *vma,
174 unsigned long address, hugepd_t hpd,
175 int flags, int pdshift);
David Gibson63551ae2005-06-21 17:14:44 -0700176struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
Naoya Horiguchie66f17f2015-02-11 15:25:22 -0800177 pmd_t *pmd, int flags);
Andi Kleenceb86872008-07-23 21:27:50 -0700178struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
Naoya Horiguchie66f17f2015-02-11 15:25:22 -0800179 pud_t *pud, int flags);
Anshuman Khandualfaaa5b62017-07-06 15:38:50 -0700180struct page *follow_huge_pgd(struct mm_struct *mm, unsigned long address,
181 pgd_t *pgd, int flags);
182
David Gibson63551ae2005-06-21 17:14:44 -0700183int pmd_huge(pmd_t pmd);
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300184int pud_huge(pud_t pud);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100185unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800186 unsigned long address, unsigned long end, pgprot_t newprot);
David Gibson63551ae2005-06-21 17:14:44 -0700187
Aneesh Kumar K.Vd5ed7442017-07-06 15:38:47 -0700188bool is_hugetlb_entry_migration(pte_t pte);
Michal Hockoab5ac902018-01-31 16:20:48 -0800189
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190#else /* !CONFIG_HUGETLB_PAGE */
191
Mel Gormana1e78772008-07-23 21:27:23 -0700192static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
193{
194}
195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196static inline unsigned long hugetlb_total_pages(void)
197{
198 return 0;
199}
200
Mike Kravetzc0d03812020-04-01 21:11:05 -0700201static inline struct address_space *hugetlb_page_mapping_lock_write(
202 struct page *hpage)
203{
204 return NULL;
205}
206
Mike Kravetz34ae2042020-08-11 18:31:38 -0700207static inline int huge_pmd_unshare(struct mm_struct *mm,
208 struct vm_area_struct *vma,
209 unsigned long *addr, pte_t *ptep)
Mike Kravetz017b1662018-10-05 15:51:29 -0700210{
211 return 0;
212}
213
214static inline void adjust_range_if_pmd_sharing_possible(
215 struct vm_area_struct *vma,
216 unsigned long *start, unsigned long *end)
217{
218}
219
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800220static inline long follow_hugetlb_page(struct mm_struct *mm,
221 struct vm_area_struct *vma, struct page **pages,
222 struct vm_area_struct **vmas, unsigned long *position,
223 unsigned long *nr_pages, long i, unsigned int flags,
224 int *nonblocking)
225{
226 BUG();
227 return 0;
228}
229
230static inline struct page *follow_huge_addr(struct mm_struct *mm,
231 unsigned long address, int write)
232{
233 return ERR_PTR(-EINVAL);
234}
235
236static inline int copy_hugetlb_page_range(struct mm_struct *dst,
237 struct mm_struct *src, struct vm_area_struct *vma)
238{
239 BUG();
240 return 0;
241}
242
Alexey Dobriyane1759c22008-10-15 23:50:22 +0400243static inline void hugetlb_report_meminfo(struct seq_file *m)
244{
245}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800246
247static inline int hugetlb_report_node_meminfo(int nid, char *buf)
248{
249 return 0;
250}
251
David Rientjes949f7ec2013-04-29 15:07:48 -0700252static inline void hugetlb_show_meminfo(void)
253{
254}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800255
256static inline struct page *follow_huge_pd(struct vm_area_struct *vma,
257 unsigned long address, hugepd_t hpd, int flags,
258 int pdshift)
259{
260 return NULL;
261}
262
263static inline struct page *follow_huge_pmd(struct mm_struct *mm,
264 unsigned long address, pmd_t *pmd, int flags)
265{
266 return NULL;
267}
268
269static inline struct page *follow_huge_pud(struct mm_struct *mm,
270 unsigned long address, pud_t *pud, int flags)
271{
272 return NULL;
273}
274
275static inline struct page *follow_huge_pgd(struct mm_struct *mm,
276 unsigned long address, pgd_t *pgd, int flags)
277{
278 return NULL;
279}
280
281static inline int prepare_hugepage_range(struct file *file,
282 unsigned long addr, unsigned long len)
283{
284 return -EINVAL;
285}
286
287static inline int pmd_huge(pmd_t pmd)
288{
289 return 0;
290}
291
292static inline int pud_huge(pud_t pud)
293{
294 return 0;
295}
296
297static inline int is_hugepage_only_range(struct mm_struct *mm,
298 unsigned long addr, unsigned long len)
299{
300 return 0;
301}
302
303static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
304 unsigned long addr, unsigned long end,
305 unsigned long floor, unsigned long ceiling)
306{
307 BUG();
308}
309
310static inline int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
311 pte_t *dst_pte,
312 struct vm_area_struct *dst_vma,
313 unsigned long dst_addr,
314 unsigned long src_addr,
315 struct page **pagep)
316{
317 BUG();
318 return 0;
319}
320
321static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
322 unsigned long sz)
323{
324 return NULL;
325}
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700326
Naoya Horiguchif40386a2013-12-12 17:12:19 -0800327static inline bool isolate_huge_page(struct page *page, struct list_head *list)
328{
329 return false;
330}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800332static inline void putback_active_hugepage(struct page *page)
333{
334}
335
336static inline void move_hugetlb_state(struct page *oldpage,
337 struct page *newpage, int reason)
338{
339}
340
341static inline unsigned long hugetlb_change_protection(
342 struct vm_area_struct *vma, unsigned long address,
343 unsigned long end, pgprot_t newprot)
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100344{
345 return 0;
346}
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800347
Mel Gormand8333522012-07-31 16:46:20 -0700348static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
349 struct vm_area_struct *vma, unsigned long start,
350 unsigned long end, struct page *ref_page)
351{
352 BUG();
353}
354
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700355static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
356 struct vm_area_struct *vma, unsigned long start,
357 unsigned long end, struct page *ref_page)
358{
359 BUG();
360}
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800361
Souptick Joardera953e772019-03-28 20:43:51 -0700362static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
Mike Kravetz1f9dccb2019-11-30 17:56:40 -0800363 struct vm_area_struct *vma, unsigned long address,
364 unsigned int flags)
Souptick Joardera953e772019-03-28 20:43:51 -0700365{
366 BUG();
367 return 0;
368}
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700369
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370#endif /* !CONFIG_HUGETLB_PAGE */
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +0530371/*
372 * hugepages at page global directory. If arch support
373 * hugepages at pgd level, they need to define this.
374 */
375#ifndef pgd_huge
376#define pgd_huge(x) 0
377#endif
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300378#ifndef p4d_huge
379#define p4d_huge(x) 0
380#endif
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +0530381
382#ifndef pgd_write
383static inline int pgd_write(pgd_t pgd)
384{
385 BUG();
386 return 0;
387}
388#endif
389
Eric B Munson4e527802009-09-21 17:03:47 -0700390#define HUGETLB_ANON_FILE "anon_hugepage"
391
Eric B Munson6bfde052009-09-21 17:03:43 -0700392enum {
393 /*
394 * The file will be used as an shm file so shmfs accounting rules
395 * apply
396 */
397 HUGETLB_SHMFS_INODE = 1,
Eric B Munson4e527802009-09-21 17:03:47 -0700398 /*
399 * The file is being created on the internal vfs mount and shmfs
400 * accounting rules do not apply
401 */
402 HUGETLB_ANONHUGE_INODE = 2,
Eric B Munson6bfde052009-09-21 17:03:43 -0700403};
404
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405#ifdef CONFIG_HUGETLBFS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406struct hugetlbfs_sb_info {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 long max_inodes; /* inodes allowed */
408 long free_inodes; /* inodes free */
409 spinlock_t stat_lock;
Andi Kleena137e1c2008-07-23 21:27:43 -0700410 struct hstate *hstate;
David Gibson90481622012-03-21 16:34:12 -0700411 struct hugepage_subpool *spool;
David Howells4a252202017-07-05 16:24:18 +0100412 kuid_t uid;
413 kgid_t gid;
414 umode_t mode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415};
416
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
418{
419 return sb->s_fs_info;
420}
421
Marc-André Lureauda14c1e2018-01-31 16:19:22 -0800422struct hugetlbfs_inode_info {
423 struct shared_policy policy;
424 struct inode vfs_inode;
Marc-André Lureauff62a342018-01-31 16:19:25 -0800425 unsigned int seals;
Marc-André Lureauda14c1e2018-01-31 16:19:22 -0800426};
427
428static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
429{
430 return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
431}
432
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -0800433extern const struct file_operations hugetlbfs_file_operations;
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +0400434extern const struct vm_operations_struct hugetlb_vm_ops;
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700435struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
Andi Kleen42d73952012-12-11 16:01:34 -0800436 struct user_struct **user, int creat_flags,
437 int page_size_log);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
Yaowei Bai719ff3212016-01-14 15:18:51 -0800439static inline bool is_file_hugepages(struct file *file)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440{
Adam Litke516dffd2007-03-01 15:46:08 -0800441 if (file->f_op == &hugetlbfs_file_operations)
Yaowei Bai719ff3212016-01-14 15:18:51 -0800442 return true;
Adam Litke516dffd2007-03-01 15:46:08 -0800443
Yaowei Bai719ff3212016-01-14 15:18:51 -0800444 return is_file_shm_hugepages(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445}
446
Christophe Leroybb297bb2020-04-01 21:11:54 -0700447static inline struct hstate *hstate_inode(struct inode *i)
448{
449 return HUGETLBFS_SB(i->i_sb)->hstate;
450}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451#else /* !CONFIG_HUGETLBFS */
452
Yaowei Bai719ff3212016-01-14 15:18:51 -0800453#define is_file_hugepages(file) false
Steven Truelove40716e22012-03-21 16:34:14 -0700454static inline struct file *
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700455hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
456 struct user_struct **user, int creat_flags,
Andi Kleen42d73952012-12-11 16:01:34 -0800457 int page_size_log)
Andrew Mortone9ea0e22009-09-24 14:47:45 -0700458{
459 return ERR_PTR(-ENOSYS);
460}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461
Christophe Leroybb297bb2020-04-01 21:11:54 -0700462static inline struct hstate *hstate_inode(struct inode *i)
463{
464 return NULL;
465}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466#endif /* !CONFIG_HUGETLBFS */
467
Adrian Bunkd2ba27e82007-05-06 14:49:00 -0700468#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
469unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
470 unsigned long len, unsigned long pgoff,
471 unsigned long flags);
472#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
473
Andi Kleena5516432008-07-23 21:27:41 -0700474#ifdef CONFIG_HUGETLB_PAGE
475
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700476#define HSTATE_NAME_LEN 32
Andi Kleena5516432008-07-23 21:27:41 -0700477/* Defines one hugetlb page size */
478struct hstate {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700479 int next_nid_to_alloc;
480 int next_nid_to_free;
Andi Kleena5516432008-07-23 21:27:41 -0700481 unsigned int order;
482 unsigned long mask;
483 unsigned long max_huge_pages;
484 unsigned long nr_huge_pages;
485 unsigned long free_huge_pages;
486 unsigned long resv_huge_pages;
487 unsigned long surplus_huge_pages;
488 unsigned long nr_overcommit_huge_pages;
Aneesh Kumar K.V0edaecf2012-07-31 16:42:07 -0700489 struct list_head hugepage_activelist;
Andi Kleena5516432008-07-23 21:27:41 -0700490 struct list_head hugepage_freelists[MAX_NUMNODES];
491 unsigned int nr_huge_pages_node[MAX_NUMNODES];
492 unsigned int free_huge_pages_node[MAX_NUMNODES];
493 unsigned int surplus_huge_pages_node[MAX_NUMNODES];
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -0700494#ifdef CONFIG_CGROUP_HUGETLB
495 /* cgroup control files */
Mina Almasrycdc2fcf2020-04-01 21:11:11 -0700496 struct cftype cgroup_files_dfl[7];
497 struct cftype cgroup_files_legacy[9];
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -0700498#endif
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700499 char name[HSTATE_NAME_LEN];
Andi Kleena5516432008-07-23 21:27:41 -0700500};
501
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700502struct huge_bootmem_page {
503 struct list_head list;
504 struct hstate *hstate;
505};
506
Mike Kravetz70c35472015-09-08 15:01:54 -0700507struct page *alloc_huge_page(struct vm_area_struct *vma,
508 unsigned long addr, int avoid_reserve);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900509struct page *alloc_huge_page_node(struct hstate *h, int nid);
Michal Hocko3e59fcb2017-07-10 15:49:11 -0700510struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
511 nodemask_t *nmask);
Michal Hocko389c8172018-01-31 16:21:03 -0800512struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
513 unsigned long address);
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -0800514struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
515 int nid, nodemask_t *nmask);
Mike Kravetzab76ad52015-09-08 15:01:50 -0700516int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
517 pgoff_t idx);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900518
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700519/* arch callback */
Aneesh Kumar K.Ve24a1302017-07-28 10:31:25 +0530520int __init __alloc_bootmem_huge_page(struct hstate *h);
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700521int __init alloc_bootmem_huge_page(struct hstate *h);
522
Andi Kleene5ff2152008-07-23 21:27:42 -0700523void __init hugetlb_add_hstate(unsigned order);
Mike Kravetzae94da82020-06-03 16:00:34 -0700524bool __init arch_hugetlb_valid_size(unsigned long size);
Andi Kleene5ff2152008-07-23 21:27:42 -0700525struct hstate *size_to_hstate(unsigned long size);
526
527#ifndef HUGE_MAX_HSTATE
528#define HUGE_MAX_HSTATE 1
529#endif
530
531extern struct hstate hstates[HUGE_MAX_HSTATE];
532extern unsigned int default_hstate_idx;
533
534#define default_hstate (hstates[default_hstate_idx])
Andi Kleena5516432008-07-23 21:27:41 -0700535
Andi Kleena5516432008-07-23 21:27:41 -0700536static inline struct hstate *hstate_file(struct file *f)
537{
Al Viro496ad9a2013-01-23 17:07:38 -0500538 return hstate_inode(file_inode(f));
Andi Kleena5516432008-07-23 21:27:41 -0700539}
540
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700541static inline struct hstate *hstate_sizelog(int page_size_log)
542{
543 if (!page_size_log)
544 return &default_hstate;
Sasha Levin97ad2be2014-12-10 15:44:13 -0800545
546 return size_to_hstate(1UL << page_size_log);
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700547}
548
Andi Kleena137e1c2008-07-23 21:27:43 -0700549static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
Andi Kleena5516432008-07-23 21:27:41 -0700550{
Andi Kleena137e1c2008-07-23 21:27:43 -0700551 return hstate_file(vma->vm_file);
Andi Kleena5516432008-07-23 21:27:41 -0700552}
553
554static inline unsigned long huge_page_size(struct hstate *h)
555{
556 return (unsigned long)PAGE_SIZE << h->order;
557}
558
Mel Gorman08fba692009-01-06 14:38:53 -0800559extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
560
Mel Gorman33402892009-01-06 14:38:54 -0800561extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
562
Andi Kleena5516432008-07-23 21:27:41 -0700563static inline unsigned long huge_page_mask(struct hstate *h)
564{
565 return h->mask;
566}
567
568static inline unsigned int huge_page_order(struct hstate *h)
569{
570 return h->order;
571}
572
573static inline unsigned huge_page_shift(struct hstate *h)
574{
575 return h->order + PAGE_SHIFT;
576}
577
Luiz Capitulinobae7f4a2014-06-04 16:07:08 -0700578static inline bool hstate_is_gigantic(struct hstate *h)
579{
580 return huge_page_order(h) >= MAX_ORDER;
581}
582
Andi Kleena5516432008-07-23 21:27:41 -0700583static inline unsigned int pages_per_huge_page(struct hstate *h)
584{
585 return 1 << h->order;
586}
587
588static inline unsigned int blocks_per_huge_page(struct hstate *h)
589{
590 return huge_page_size(h) / 512;
591}
592
593#include <asm/hugetlb.h>
594
Anshuman Khandualb0eae982020-06-03 16:01:01 -0700595#ifndef is_hugepage_only_range
596static inline int is_hugepage_only_range(struct mm_struct *mm,
597 unsigned long addr, unsigned long len)
598{
599 return 0;
600}
601#define is_hugepage_only_range is_hugepage_only_range
602#endif
603
Anshuman Khandual5be99342020-06-03 16:01:05 -0700604#ifndef arch_clear_hugepage_flags
605static inline void arch_clear_hugepage_flags(struct page *page) { }
606#define arch_clear_hugepage_flags arch_clear_hugepage_flags
607#endif
608
Chris Metcalfd9ed9fa2012-04-01 14:01:34 -0400609#ifndef arch_make_huge_pte
610static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
611 struct page *page, int writable)
612{
613 return entry;
614}
615#endif
616
Andi Kleene5ff2152008-07-23 21:27:42 -0700617static inline struct hstate *page_hstate(struct page *page)
618{
Sasha Levin309381fea2014-01-23 15:52:54 -0800619 VM_BUG_ON_PAGE(!PageHuge(page), page);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -0700620 return size_to_hstate(page_size(page));
Andi Kleene5ff2152008-07-23 21:27:42 -0700621}
622
Andi Kleenaa50d3a2010-10-06 21:45:00 +0200623static inline unsigned hstate_index_to_shift(unsigned index)
624{
625 return hstates[index].order + PAGE_SHIFT;
626}
627
Aneesh Kumar K.V972dc4d2012-07-31 16:42:00 -0700628static inline int hstate_index(struct hstate *h)
629{
630 return h - hstates;
631}
632
Zhang Yi13d60f42013-06-25 21:19:31 +0800633pgoff_t __basepage_index(struct page *page);
634
635/* Return page->index in PAGE_SIZE units */
636static inline pgoff_t basepage_index(struct page *page)
637{
638 if (!PageCompound(page))
639 return page->index;
640
641 return __basepage_index(page);
642}
643
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700644extern int dissolve_free_huge_page(struct page *page);
Gerald Schaefer082d5b62016-10-07 17:01:10 -0700645extern int dissolve_free_huge_pages(unsigned long start_pfn,
646 unsigned long end_pfn);
Anshuman Khanduale693de12019-03-05 15:43:51 -0800647
Naoya Horiguchic177c812014-06-04 16:05:35 -0700648#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
Anshuman Khanduale693de12019-03-05 15:43:51 -0800649#ifndef arch_hugetlb_migration_supported
650static inline bool arch_hugetlb_migration_supported(struct hstate *h)
651{
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700652 if ((huge_page_shift(h) == PMD_SHIFT) ||
Anshuman Khandual9b553bf2019-03-05 15:43:48 -0800653 (huge_page_shift(h) == PUD_SHIFT) ||
654 (huge_page_shift(h) == PGDIR_SHIFT))
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700655 return true;
656 else
657 return false;
Anshuman Khanduale693de12019-03-05 15:43:51 -0800658}
Naoya Horiguchic177c812014-06-04 16:05:35 -0700659#endif
Anshuman Khanduale693de12019-03-05 15:43:51 -0800660#else
661static inline bool arch_hugetlb_migration_supported(struct hstate *h)
662{
663 return false;
664}
665#endif
666
667static inline bool hugepage_migration_supported(struct hstate *h)
668{
669 return arch_hugetlb_migration_supported(h);
Naoya Horiguchi83467ef2013-09-11 14:22:11 -0700670}
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700671
Anshuman Khandual7ed2c312019-03-05 15:43:44 -0800672/*
673 * Movability check is different as compared to migration check.
674 * It determines whether or not a huge page should be placed on
675 * movable zone or not. Movability of any huge page should be
676 * required only if huge page size is supported for migration.
677 * There wont be any reason for the huge page to be movable if
678 * it is not migratable to start with. Also the size of the huge
679 * page should be large enough to be placed under a movable zone
680 * and still feasible enough to be migratable. Just the presence
681 * in movable zone does not make the migration feasible.
682 *
683 * So even though large huge page sizes like the gigantic ones
684 * are migratable they should not be movable because its not
685 * feasible to migrate them from movable zone.
686 */
687static inline bool hugepage_movable_supported(struct hstate *h)
688{
689 if (!hugepage_migration_supported(h))
690 return false;
691
692 if (hstate_is_gigantic(h))
693 return false;
694 return true;
695}
696
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800697static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
698 struct mm_struct *mm, pte_t *pte)
699{
700 if (huge_page_size(h) == PMD_SIZE)
701 return pmd_lockptr(mm, (pmd_t *) pte);
702 VM_BUG_ON(huge_page_size(h) == PAGE_SIZE);
703 return &mm->page_table_lock;
704}
705
Dominik Dingel2531c8c2015-07-17 16:23:37 -0700706#ifndef hugepages_supported
707/*
708 * Some platform decide whether they support huge pages at boot
709 * time. Some of them, such as powerpc, set HPAGE_SHIFT to 0
710 * when there is no such support
711 */
712#define hugepages_supported() (HPAGE_SHIFT != 0)
713#endif
Nishanth Aravamudan457c1b22014-05-06 12:50:00 -0700714
Naoya Horiguchi5d317b22015-11-05 18:47:14 -0800715void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm);
716
717static inline void hugetlb_count_add(long l, struct mm_struct *mm)
718{
719 atomic_long_add(l, &mm->hugetlb_usage);
720}
721
722static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
723{
724 atomic_long_sub(l, &mm->hugetlb_usage);
725}
Punit Agrawale5251fd2017-07-06 15:39:50 -0700726
727#ifndef set_huge_swap_pte_at
728static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
729 pte_t *ptep, pte_t pte, unsigned long sz)
730{
731 set_huge_pte_at(mm, addr, ptep, pte);
732}
733#endif
Aneesh Kumar K.V023bdd02019-03-05 15:46:37 -0800734
735#ifndef huge_ptep_modify_prot_start
736#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
737static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
738 unsigned long addr, pte_t *ptep)
739{
740 return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
741}
742#endif
743
744#ifndef huge_ptep_modify_prot_commit
745#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
746static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
747 unsigned long addr, pte_t *ptep,
748 pte_t old_pte, pte_t pte)
749{
750 set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
751}
752#endif
753
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700754#else /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700755struct hstate {};
Jason Gunthorpe442a5a92019-07-11 20:54:40 -0700756
757static inline struct page *alloc_huge_page(struct vm_area_struct *vma,
758 unsigned long addr,
759 int avoid_reserve)
760{
761 return NULL;
762}
763
764static inline struct page *alloc_huge_page_node(struct hstate *h, int nid)
765{
766 return NULL;
767}
768
769static inline struct page *
770alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask)
771{
772 return NULL;
773}
774
775static inline struct page *alloc_huge_page_vma(struct hstate *h,
776 struct vm_area_struct *vma,
777 unsigned long address)
778{
779 return NULL;
780}
781
782static inline int __alloc_bootmem_huge_page(struct hstate *h)
783{
784 return 0;
785}
786
787static inline struct hstate *hstate_file(struct file *f)
788{
789 return NULL;
790}
791
792static inline struct hstate *hstate_sizelog(int page_size_log)
793{
794 return NULL;
795}
796
797static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
798{
799 return NULL;
800}
801
Jason Gunthorpe442a5a92019-07-11 20:54:40 -0700802static inline struct hstate *page_hstate(struct page *page)
803{
804 return NULL;
805}
806
807static inline unsigned long huge_page_size(struct hstate *h)
808{
809 return PAGE_SIZE;
810}
811
812static inline unsigned long huge_page_mask(struct hstate *h)
813{
814 return PAGE_MASK;
815}
816
817static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
818{
819 return PAGE_SIZE;
820}
821
822static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
823{
824 return PAGE_SIZE;
825}
826
827static inline unsigned int huge_page_order(struct hstate *h)
828{
829 return 0;
830}
831
832static inline unsigned int huge_page_shift(struct hstate *h)
833{
834 return PAGE_SHIFT;
835}
836
Anshuman Khandual94310cb2017-07-06 15:38:38 -0700837static inline bool hstate_is_gigantic(struct hstate *h)
838{
839 return false;
840}
841
Andrea Righi510a35d2008-07-26 15:22:27 -0700842static inline unsigned int pages_per_huge_page(struct hstate *h)
843{
844 return 1;
845}
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700846
847static inline unsigned hstate_index_to_shift(unsigned index)
848{
849 return 0;
850}
851
852static inline int hstate_index(struct hstate *h)
853{
854 return 0;
855}
Zhang Yi13d60f42013-06-25 21:19:31 +0800856
857static inline pgoff_t basepage_index(struct page *page)
858{
859 return page->index;
860}
Anshuman Khandualc3114a82017-07-10 15:47:41 -0700861
862static inline int dissolve_free_huge_page(struct page *page)
863{
864 return 0;
865}
866
867static inline int dissolve_free_huge_pages(unsigned long start_pfn,
868 unsigned long end_pfn)
869{
870 return 0;
871}
872
873static inline bool hugepage_migration_supported(struct hstate *h)
874{
875 return false;
876}
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800877
Anshuman Khandual7ed2c312019-03-05 15:43:44 -0800878static inline bool hugepage_movable_supported(struct hstate *h)
879{
880 return false;
881}
882
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800883static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
884 struct mm_struct *mm, pte_t *pte)
885{
886 return &mm->page_table_lock;
887}
Naoya Horiguchi5d317b22015-11-05 18:47:14 -0800888
889static inline void hugetlb_report_usage(struct seq_file *f, struct mm_struct *m)
890{
891}
892
893static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
894{
895}
Punit Agrawale5251fd2017-07-06 15:39:50 -0700896
897static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
898 pte_t *ptep, pte_t pte, unsigned long sz)
899{
900}
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700901#endif /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700902
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800903static inline spinlock_t *huge_pte_lock(struct hstate *h,
904 struct mm_struct *mm, pte_t *pte)
905{
906 spinlock_t *ptl;
907
908 ptl = huge_pte_lockptr(h, mm, pte);
909 spin_lock(ptl);
910 return ptl;
911}
912
Roman Gushchincf11e852020-04-10 14:32:45 -0700913#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
914extern void __init hugetlb_cma_reserve(int order);
915extern void __init hugetlb_cma_check(void);
916#else
917static inline __init void hugetlb_cma_reserve(int order)
918{
919}
920static inline __init void hugetlb_cma_check(void)
921{
922}
923#endif
924
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925#endif /* _LINUX_HUGETLB_H */