mm, page_alloc: shorten the page allocator fast path
The page allocator fast path checks page multiple times unnecessarily.
This patch avoids all the slowpath checks if the first allocation
attempt succeeds.
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a3b7eb8..8380011 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3398,31 +3398,26 @@
ac.nodemask, &ac.preferred_zone);
if (!ac.preferred_zone) {
page = NULL;
- goto out;
+ goto no_zone;
}
ac.classzone_idx = zonelist_zone_idx(preferred_zoneref);
/* First allocation attempt */
page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
- if (unlikely(!page)) {
- /*
- * Runtime PM, block IO and its error handling path
- * can deadlock because I/O on the device might not
- * complete.
- */
- alloc_mask = memalloc_noio_flags(gfp_mask);
- ac.spread_dirty_pages = false;
+ if (likely(page))
+ goto out;
- page = __alloc_pages_slowpath(alloc_mask, order, &ac);
- }
+ /*
+ * Runtime PM, block IO and its error handling path can deadlock
+ * because I/O on the device might not complete.
+ */
+ alloc_mask = memalloc_noio_flags(gfp_mask);
+ ac.spread_dirty_pages = false;
- if (kmemcheck_enabled && page)
- kmemcheck_pagealloc_alloc(page, order, gfp_mask);
+ page = __alloc_pages_slowpath(alloc_mask, order, &ac);
- trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
-
-out:
+no_zone:
/*
* When updating a task's mems_allowed, it is possible to race with
* parallel threads in such a way that an allocation can fail while
@@ -3434,6 +3429,12 @@
goto retry_cpuset;
}
+out:
+ if (kmemcheck_enabled && page)
+ kmemcheck_pagealloc_alloc(page, order, gfp_mask);
+
+ trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
+
return page;
}
EXPORT_SYMBOL(__alloc_pages_nodemask);