blob: e6b62f30ab217ea30338c2bd58c5d52264c90c91 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_HUGETLB_H
2#define _LINUX_HUGETLB_H
3
Linus Torvaldsbe93d8c2011-05-26 12:03:50 -07004#include <linux/mm_types.h>
Sasha Levin309381fea2014-01-23 15:52:54 -08005#include <linux/mmdebug.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +04006#include <linux/fs.h>
Naoya Horiguchi8edf3442010-05-28 09:29:15 +09007#include <linux/hugetlb_inline.h>
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -07008#include <linux/cgroup.h>
Joonsoo Kim9119a412014-04-03 14:47:25 -07009#include <linux/list.h>
10#include <linux/kref.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040011
Andrew Mortone9ea0e22009-09-24 14:47:45 -070012struct ctl_table;
13struct user_struct;
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -070014struct mmu_gather;
Andrew Mortone9ea0e22009-09-24 14:47:45 -070015
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#ifdef CONFIG_HUGETLB_PAGE
17
18#include <linux/mempolicy.h>
Adam Litke516dffd2007-03-01 15:46:08 -080019#include <linux/shm.h>
David Gibson63551ae2005-06-21 17:14:44 -070020#include <asm/tlbflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
David Gibson90481622012-03-21 16:34:12 -070022struct hugepage_subpool {
23 spinlock_t lock;
24 long count;
25 long max_hpages, used_hpages;
26};
27
Joonsoo Kim9119a412014-04-03 14:47:25 -070028struct resv_map {
29 struct kref refs;
Davidlohr Bueso7b24d862014-04-03 14:47:27 -070030 spinlock_t lock;
Joonsoo Kim9119a412014-04-03 14:47:25 -070031 struct list_head regions;
32};
33extern struct resv_map *resv_map_alloc(void);
34void resv_map_release(struct kref *ref);
35
Aneesh Kumar K.Vc3f38a32012-07-31 16:42:10 -070036extern spinlock_t hugetlb_lock;
37extern int hugetlb_max_hstate __read_mostly;
38#define for_each_hstate(h) \
39 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
40
David Gibson90481622012-03-21 16:34:12 -070041struct hugepage_subpool *hugepage_new_subpool(long nr_blocks);
42void hugepage_put_subpool(struct hugepage_subpool *spool);
43
Wu Fengguang20a03072009-06-16 15:32:22 -070044int PageHuge(struct page *page);
45
Mel Gormana1e78772008-07-23 21:27:23 -070046void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
Alexey Dobriyan8d65af72009-09-23 15:57:19 -070047int hugetlb_sysctl_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
48int hugetlb_overcommit_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
49int hugetlb_treat_movable_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
Lee Schermerhorn06808b02009-12-14 17:58:21 -080050
51#ifdef CONFIG_NUMA
52int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int,
53 void __user *, size_t *, loff_t *);
54#endif
55
Linus Torvalds1da177e2005-04-16 15:20:36 -070056int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
Michel Lespinasse28a35712013-02-22 16:35:55 -080057long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
58 struct page **, struct vm_area_struct **,
59 unsigned long *, unsigned long *, long, unsigned int);
Mel Gorman04f2cbe2008-07-23 21:27:25 -070060void unmap_hugepage_range(struct vm_area_struct *,
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -070061 unsigned long, unsigned long, struct page *);
Mel Gormand8333522012-07-31 16:46:20 -070062void __unmap_hugepage_range_final(struct mmu_gather *tlb,
63 struct vm_area_struct *vma,
64 unsigned long start, unsigned long end,
65 struct page *ref_page);
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -070066void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
67 unsigned long start, unsigned long end,
68 struct page *ref_page);
Alexey Dobriyane1759c22008-10-15 23:50:22 +040069void hugetlb_report_meminfo(struct seq_file *);
Linus Torvalds1da177e2005-04-16 15:20:36 -070070int hugetlb_report_node_meminfo(int, char *);
David Rientjes949f7ec2013-04-29 15:07:48 -070071void hugetlb_show_meminfo(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -070072unsigned long hugetlb_total_pages(void);
Hugh Dickinsac9b9c62005-10-20 16:24:28 +010073int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
Hugh Dickins788c7df2009-06-23 13:49:05 +010074 unsigned long address, unsigned int flags);
Mel Gormana1e78772008-07-23 21:27:23 -070075int hugetlb_reserve_pages(struct inode *inode, long from, long to,
Mel Gorman5a6fe122009-02-10 14:02:27 +000076 struct vm_area_struct *vma,
KOSAKI Motohiroca16d142011-05-26 19:16:19 +090077 vm_flags_t vm_flags);
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -070078void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed);
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +090079int dequeue_hwpoisoned_huge_page(struct page *page);
Naoya Horiguchi31caf662013-09-11 14:21:59 -070080bool isolate_huge_page(struct page *page, struct list_head *list);
81void putback_active_hugepage(struct page *page);
Naoya Horiguchic8721bb2013-09-11 14:22:09 -070082bool is_hugepage_active(struct page *page);
Atsushi Kumagai8f1d26d2014-07-30 16:08:39 -070083void free_huge_page(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -070084
Steve Capper3212b532013-04-23 12:35:02 +010085#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
86pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
87#endif
88
Mel Gorman396faf02007-07-17 04:03:13 -070089extern unsigned long hugepages_treat_as_movable;
Linus Torvalds1da177e2005-04-16 15:20:36 -070090extern int sysctl_hugetlb_shm_group;
Jon Tollefson53ba51d2008-07-23 21:27:52 -070091extern struct list_head huge_boot_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -070092
David Gibson63551ae2005-06-21 17:14:44 -070093/* arch callbacks */
94
Andi Kleena5516432008-07-23 21:27:41 -070095pte_t *huge_pte_alloc(struct mm_struct *mm,
96 unsigned long addr, unsigned long sz);
David Gibson63551ae2005-06-21 17:14:44 -070097pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr);
Chen, Kenneth W39dde652006-12-06 20:32:03 -080098int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep);
David Gibson63551ae2005-06-21 17:14:44 -070099struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
100 int write);
101struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
102 pmd_t *pmd, int write);
Andi Kleenceb86872008-07-23 21:27:50 -0700103struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
104 pud_t *pud, int write);
David Gibson63551ae2005-06-21 17:14:44 -0700105int pmd_huge(pmd_t pmd);
Andi Kleenceb86872008-07-23 21:27:50 -0700106int pud_huge(pud_t pmd);
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100107unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800108 unsigned long address, unsigned long end, pgprot_t newprot);
David Gibson63551ae2005-06-21 17:14:44 -0700109
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110#else /* !CONFIG_HUGETLB_PAGE */
111
Wu Fengguang20a03072009-06-16 15:32:22 -0700112static inline int PageHuge(struct page *page)
113{
114 return 0;
115}
116
Mel Gormana1e78772008-07-23 21:27:23 -0700117static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
118{
119}
120
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121static inline unsigned long hugetlb_total_pages(void)
122{
123 return 0;
124}
125
Adam Litke5b23dbe2007-11-14 16:59:33 -0800126#define follow_hugetlb_page(m,v,p,vs,a,b,i,w) ({ BUG(); 0; })
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127#define follow_huge_addr(mm, addr, write) ERR_PTR(-EINVAL)
128#define copy_hugetlb_page_range(src, dst, vma) ({ BUG(); 0; })
Alexey Dobriyane1759c22008-10-15 23:50:22 +0400129static inline void hugetlb_report_meminfo(struct seq_file *m)
130{
131}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132#define hugetlb_report_node_meminfo(n, buf) 0
David Rientjes949f7ec2013-04-29 15:07:48 -0700133static inline void hugetlb_show_meminfo(void)
134{
135}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136#define follow_huge_pmd(mm, addr, pmd, write) NULL
Andi Kleenceb86872008-07-23 21:27:50 -0700137#define follow_huge_pud(mm, addr, pud, write) NULL
Andi Kleena5516432008-07-23 21:27:41 -0700138#define prepare_hugepage_range(file, addr, len) (-EINVAL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139#define pmd_huge(x) 0
Andi Kleenceb86872008-07-23 21:27:50 -0700140#define pud_huge(x) 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141#define is_hugepage_only_range(mm, addr, len) 0
David Gibson9da61ae2006-03-22 00:08:57 -0800142#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; })
Hugh Dickins788c7df2009-06-23 13:49:05 +0100143#define hugetlb_fault(mm, vma, addr, flags) ({ BUG(); 0; })
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +0900144#define huge_pte_offset(mm, address) 0
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700145static inline int dequeue_hwpoisoned_huge_page(struct page *page)
146{
147 return 0;
148}
149
Naoya Horiguchif40386a2013-12-12 17:12:19 -0800150static inline bool isolate_huge_page(struct page *page, struct list_head *list)
151{
152 return false;
153}
Naoya Horiguchi31caf662013-09-11 14:21:59 -0700154#define putback_active_hugepage(p) do {} while (0)
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700155#define is_hugepage_active(x) false
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156
Peter Zijlstra7da4d642012-11-19 03:14:23 +0100157static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
158 unsigned long address, unsigned long end, pgprot_t newprot)
159{
160 return 0;
161}
Zhang, Yanmin8f860592006-03-22 00:08:50 -0800162
Mel Gormand8333522012-07-31 16:46:20 -0700163static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
164 struct vm_area_struct *vma, unsigned long start,
165 unsigned long end, struct page *ref_page)
166{
167 BUG();
168}
169
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700170static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
171 struct vm_area_struct *vma, unsigned long start,
172 unsigned long end, struct page *ref_page)
173{
174 BUG();
175}
176
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177#endif /* !CONFIG_HUGETLB_PAGE */
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +0530178/*
179 * hugepages at page global directory. If arch support
180 * hugepages at pgd level, they need to define this.
181 */
182#ifndef pgd_huge
183#define pgd_huge(x) 0
184#endif
185
186#ifndef pgd_write
187static inline int pgd_write(pgd_t pgd)
188{
189 BUG();
190 return 0;
191}
192#endif
193
194#ifndef pud_write
195static inline int pud_write(pud_t pud)
196{
197 BUG();
198 return 0;
199}
200#endif
201
202#ifndef is_hugepd
203/*
204 * Some architectures requires a hugepage directory format that is
205 * required to support multiple hugepage sizes. For example
206 * a4fe3ce76 "powerpc/mm: Allow more flexible layouts for hugepage pagetables"
207 * introduced the same on powerpc. This allows for a more flexible hugepage
208 * pagetable layout.
209 */
210typedef struct { unsigned long pd; } hugepd_t;
211#define is_hugepd(hugepd) (0)
212#define __hugepd(x) ((hugepd_t) { (x) })
213static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
214 unsigned pdshift, unsigned long end,
215 int write, struct page **pages, int *nr)
216{
217 return 0;
218}
219#else
220extern int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
221 unsigned pdshift, unsigned long end,
222 int write, struct page **pages, int *nr);
223#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224
Eric B Munson4e527802009-09-21 17:03:47 -0700225#define HUGETLB_ANON_FILE "anon_hugepage"
226
Eric B Munson6bfde052009-09-21 17:03:43 -0700227enum {
228 /*
229 * The file will be used as an shm file so shmfs accounting rules
230 * apply
231 */
232 HUGETLB_SHMFS_INODE = 1,
Eric B Munson4e527802009-09-21 17:03:47 -0700233 /*
234 * The file is being created on the internal vfs mount and shmfs
235 * accounting rules do not apply
236 */
237 HUGETLB_ANONHUGE_INODE = 2,
Eric B Munson6bfde052009-09-21 17:03:43 -0700238};
239
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240#ifdef CONFIG_HUGETLBFS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241struct hugetlbfs_sb_info {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 long max_inodes; /* inodes allowed */
243 long free_inodes; /* inodes free */
244 spinlock_t stat_lock;
Andi Kleena137e1c2008-07-23 21:27:43 -0700245 struct hstate *hstate;
David Gibson90481622012-03-21 16:34:12 -0700246 struct hugepage_subpool *spool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247};
248
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
250{
251 return sb->s_fs_info;
252}
253
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -0800254extern const struct file_operations hugetlbfs_file_operations;
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +0400255extern const struct vm_operations_struct hugetlb_vm_ops;
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700256struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
Andi Kleen42d73952012-12-11 16:01:34 -0800257 struct user_struct **user, int creat_flags,
258 int page_size_log);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
260static inline int is_file_hugepages(struct file *file)
261{
Adam Litke516dffd2007-03-01 15:46:08 -0800262 if (file->f_op == &hugetlbfs_file_operations)
263 return 1;
264 if (is_file_shm_hugepages(file))
265 return 1;
266
267 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268}
269
Andi Kleen42d73952012-12-11 16:01:34 -0800270
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271#else /* !CONFIG_HUGETLBFS */
272
Stefan Richter1db85082009-02-10 23:27:32 +0100273#define is_file_hugepages(file) 0
Steven Truelove40716e22012-03-21 16:34:14 -0700274static inline struct file *
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700275hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
276 struct user_struct **user, int creat_flags,
Andi Kleen42d73952012-12-11 16:01:34 -0800277 int page_size_log)
Andrew Mortone9ea0e22009-09-24 14:47:45 -0700278{
279 return ERR_PTR(-ENOSYS);
280}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281
282#endif /* !CONFIG_HUGETLBFS */
283
Adrian Bunkd2ba27e82007-05-06 14:49:00 -0700284#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
285unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
286 unsigned long len, unsigned long pgoff,
287 unsigned long flags);
288#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
289
Andi Kleena5516432008-07-23 21:27:41 -0700290#ifdef CONFIG_HUGETLB_PAGE
291
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700292#define HSTATE_NAME_LEN 32
Andi Kleena5516432008-07-23 21:27:41 -0700293/* Defines one hugetlb page size */
294struct hstate {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700295 int next_nid_to_alloc;
296 int next_nid_to_free;
Andi Kleena5516432008-07-23 21:27:41 -0700297 unsigned int order;
298 unsigned long mask;
299 unsigned long max_huge_pages;
300 unsigned long nr_huge_pages;
301 unsigned long free_huge_pages;
302 unsigned long resv_huge_pages;
303 unsigned long surplus_huge_pages;
304 unsigned long nr_overcommit_huge_pages;
Aneesh Kumar K.V0edaecf2012-07-31 16:42:07 -0700305 struct list_head hugepage_activelist;
Andi Kleena5516432008-07-23 21:27:41 -0700306 struct list_head hugepage_freelists[MAX_NUMNODES];
307 unsigned int nr_huge_pages_node[MAX_NUMNODES];
308 unsigned int free_huge_pages_node[MAX_NUMNODES];
309 unsigned int surplus_huge_pages_node[MAX_NUMNODES];
Aneesh Kumar K.Vabb82062012-07-31 16:42:24 -0700310#ifdef CONFIG_CGROUP_HUGETLB
311 /* cgroup control files */
312 struct cftype cgroup_files[5];
313#endif
Nishanth Aravamudana3437872008-07-23 21:27:44 -0700314 char name[HSTATE_NAME_LEN];
Andi Kleena5516432008-07-23 21:27:41 -0700315};
316
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700317struct huge_bootmem_page {
318 struct list_head list;
319 struct hstate *hstate;
Becky Bruceee8f2482011-07-25 17:11:50 -0700320#ifdef CONFIG_HIGHMEM
321 phys_addr_t phys;
322#endif
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700323};
324
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900325struct page *alloc_huge_page_node(struct hstate *h, int nid);
Naoya Horiguchi74060e42013-09-11 14:22:06 -0700326struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
327 unsigned long addr, int avoid_reserve);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900328
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700329/* arch callback */
330int __init alloc_bootmem_huge_page(struct hstate *h);
331
Andi Kleene5ff2152008-07-23 21:27:42 -0700332void __init hugetlb_add_hstate(unsigned order);
333struct hstate *size_to_hstate(unsigned long size);
334
335#ifndef HUGE_MAX_HSTATE
336#define HUGE_MAX_HSTATE 1
337#endif
338
339extern struct hstate hstates[HUGE_MAX_HSTATE];
340extern unsigned int default_hstate_idx;
341
342#define default_hstate (hstates[default_hstate_idx])
Andi Kleena5516432008-07-23 21:27:41 -0700343
Andi Kleena137e1c2008-07-23 21:27:43 -0700344static inline struct hstate *hstate_inode(struct inode *i)
Andi Kleena5516432008-07-23 21:27:41 -0700345{
Andi Kleena137e1c2008-07-23 21:27:43 -0700346 struct hugetlbfs_sb_info *hsb;
347 hsb = HUGETLBFS_SB(i->i_sb);
348 return hsb->hstate;
Andi Kleena5516432008-07-23 21:27:41 -0700349}
350
351static inline struct hstate *hstate_file(struct file *f)
352{
Al Viro496ad9a2013-01-23 17:07:38 -0500353 return hstate_inode(file_inode(f));
Andi Kleena5516432008-07-23 21:27:41 -0700354}
355
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700356static inline struct hstate *hstate_sizelog(int page_size_log)
357{
358 if (!page_size_log)
359 return &default_hstate;
360 return size_to_hstate(1 << page_size_log);
361}
362
Andi Kleena137e1c2008-07-23 21:27:43 -0700363static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
Andi Kleena5516432008-07-23 21:27:41 -0700364{
Andi Kleena137e1c2008-07-23 21:27:43 -0700365 return hstate_file(vma->vm_file);
Andi Kleena5516432008-07-23 21:27:41 -0700366}
367
368static inline unsigned long huge_page_size(struct hstate *h)
369{
370 return (unsigned long)PAGE_SIZE << h->order;
371}
372
Mel Gorman08fba692009-01-06 14:38:53 -0800373extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
374
Mel Gorman33402892009-01-06 14:38:54 -0800375extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
376
Andi Kleena5516432008-07-23 21:27:41 -0700377static inline unsigned long huge_page_mask(struct hstate *h)
378{
379 return h->mask;
380}
381
382static inline unsigned int huge_page_order(struct hstate *h)
383{
384 return h->order;
385}
386
387static inline unsigned huge_page_shift(struct hstate *h)
388{
389 return h->order + PAGE_SHIFT;
390}
391
Luiz Capitulinobae7f4a2014-06-04 16:07:08 -0700392static inline bool hstate_is_gigantic(struct hstate *h)
393{
394 return huge_page_order(h) >= MAX_ORDER;
395}
396
Andi Kleena5516432008-07-23 21:27:41 -0700397static inline unsigned int pages_per_huge_page(struct hstate *h)
398{
399 return 1 << h->order;
400}
401
402static inline unsigned int blocks_per_huge_page(struct hstate *h)
403{
404 return huge_page_size(h) / 512;
405}
406
407#include <asm/hugetlb.h>
408
Chris Metcalfd9ed9fa2012-04-01 14:01:34 -0400409#ifndef arch_make_huge_pte
410static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
411 struct page *page, int writable)
412{
413 return entry;
414}
415#endif
416
Andi Kleene5ff2152008-07-23 21:27:42 -0700417static inline struct hstate *page_hstate(struct page *page)
418{
Sasha Levin309381fea2014-01-23 15:52:54 -0800419 VM_BUG_ON_PAGE(!PageHuge(page), page);
Andi Kleene5ff2152008-07-23 21:27:42 -0700420 return size_to_hstate(PAGE_SIZE << compound_order(page));
421}
422
Andi Kleenaa50d3a2010-10-06 21:45:00 +0200423static inline unsigned hstate_index_to_shift(unsigned index)
424{
425 return hstates[index].order + PAGE_SHIFT;
426}
427
Aneesh Kumar K.V972dc4d2012-07-31 16:42:00 -0700428static inline int hstate_index(struct hstate *h)
429{
430 return h - hstates;
431}
432
Zhang Yi13d60f42013-06-25 21:19:31 +0800433pgoff_t __basepage_index(struct page *page);
434
435/* Return page->index in PAGE_SIZE units */
436static inline pgoff_t basepage_index(struct page *page)
437{
438 if (!PageCompound(page))
439 return page->index;
440
441 return __basepage_index(page);
442}
443
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700444extern void dissolve_free_huge_pages(unsigned long start_pfn,
445 unsigned long end_pfn);
Naoya Horiguchi100873d2014-06-04 16:10:56 -0700446static inline int hugepage_migration_supported(struct hstate *h)
Naoya Horiguchi83467ef2013-09-11 14:22:11 -0700447{
Naoya Horiguchic177c812014-06-04 16:05:35 -0700448#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
449 return huge_page_shift(h) == PMD_SHIFT;
450#else
451 return 0;
452#endif
Naoya Horiguchi83467ef2013-09-11 14:22:11 -0700453}
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700454
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800455static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
456 struct mm_struct *mm, pte_t *pte)
457{
458 if (huge_page_size(h) == PMD_SIZE)
459 return pmd_lockptr(mm, (pmd_t *) pte);
460 VM_BUG_ON(huge_page_size(h) == PAGE_SIZE);
461 return &mm->page_table_lock;
462}
463
Nishanth Aravamudan457c1b22014-05-06 12:50:00 -0700464static inline bool hugepages_supported(void)
465{
466 /*
467 * Some platform decide whether they support huge pages at boot
468 * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
469 * there is no such support
470 */
471 return HPAGE_SHIFT != 0;
472}
473
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700474#else /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700475struct hstate {};
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900476#define alloc_huge_page_node(h, nid) NULL
Naoya Horiguchi74060e42013-09-11 14:22:06 -0700477#define alloc_huge_page_noerr(v, a, r) NULL
Jon Tollefson53ba51d2008-07-23 21:27:52 -0700478#define alloc_bootmem_huge_page(h) NULL
Andi Kleena5516432008-07-23 21:27:41 -0700479#define hstate_file(f) NULL
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700480#define hstate_sizelog(s) NULL
Andi Kleena5516432008-07-23 21:27:41 -0700481#define hstate_vma(v) NULL
482#define hstate_inode(i) NULL
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800483#define page_hstate(page) NULL
Andi Kleena5516432008-07-23 21:27:41 -0700484#define huge_page_size(h) PAGE_SIZE
485#define huge_page_mask(h) PAGE_MASK
Mel Gorman08fba692009-01-06 14:38:53 -0800486#define vma_kernel_pagesize(v) PAGE_SIZE
Mel Gorman33402892009-01-06 14:38:54 -0800487#define vma_mmu_pagesize(v) PAGE_SIZE
Andi Kleena5516432008-07-23 21:27:41 -0700488#define huge_page_order(h) 0
489#define huge_page_shift(h) PAGE_SHIFT
Andrea Righi510a35d2008-07-26 15:22:27 -0700490static inline unsigned int pages_per_huge_page(struct hstate *h)
491{
492 return 1;
493}
Andi Kleenaa50d3a2010-10-06 21:45:00 +0200494#define hstate_index_to_shift(index) 0
Aneesh Kumar K.V972dc4d2012-07-31 16:42:00 -0700495#define hstate_index(h) 0
Zhang Yi13d60f42013-06-25 21:19:31 +0800496
497static inline pgoff_t basepage_index(struct page *page)
498{
499 return page->index;
500}
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700501#define dissolve_free_huge_pages(s, e) do {} while (0)
Naoya Horiguchi100873d2014-06-04 16:10:56 -0700502#define hugepage_migration_supported(h) 0
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800503
504static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
505 struct mm_struct *mm, pte_t *pte)
506{
507 return &mm->page_table_lock;
508}
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700509#endif /* CONFIG_HUGETLB_PAGE */
Andi Kleena5516432008-07-23 21:27:41 -0700510
Kirill A. Shutemovcb900f42013-11-14 14:31:02 -0800511static inline spinlock_t *huge_pte_lock(struct hstate *h,
512 struct mm_struct *mm, pte_t *pte)
513{
514 spinlock_t *ptl;
515
516 ptl = huge_pte_lockptr(h, mm, pte);
517 spin_lock(ptl);
518 return ptl;
519}
520
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521#endif /* _LINUX_HUGETLB_H */