blob: f74386293632da785edab5cbd699d5bd15c1ecdb [file] [log] [blame]
Matthew Wilcoxd475c632015-02-16 15:58:56 -08001/*
2 * fs/dax.c - Direct Access filesystem code
3 * Copyright (c) 2013-2014 Intel Corporation
4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 */
16
17#include <linux/atomic.h>
18#include <linux/blkdev.h>
19#include <linux/buffer_head.h>
Ross Zwislerd77e92e2015-09-09 10:29:40 -060020#include <linux/dax.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080021#include <linux/fs.h>
22#include <linux/genhd.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080023#include <linux/highmem.h>
24#include <linux/memcontrol.h>
25#include <linux/mm.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080026#include <linux/mutex.h>
Ross Zwisler9973c982016-01-22 15:10:47 -080027#include <linux/pagevec.h>
Matthew Wilcox289c6ae2015-02-16 15:58:59 -080028#include <linux/sched.h>
Ingo Molnarf361bf42017-02-03 23:47:37 +010029#include <linux/sched/signal.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080030#include <linux/uio.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080031#include <linux/vmstat.h>
Dan Williams34c0fd52016-01-15 16:56:14 -080032#include <linux/pfn_t.h>
Dan Williams0e749e52016-01-15 16:55:53 -080033#include <linux/sizes.h>
Jan Kara4b4bb462016-12-14 15:07:53 -080034#include <linux/mmu_notifier.h>
Christoph Hellwiga254e562016-09-19 11:24:49 +100035#include <linux/iomap.h>
Aneesh Kumar K.V11cf9d82019-03-09 17:37:21 +053036#include <asm/pgalloc.h>
Christoph Hellwiga254e562016-09-19 11:24:49 +100037#include "internal.h"
Matthew Wilcoxd475c632015-02-16 15:58:56 -080038
Ross Zwisler282a8e02017-02-22 15:39:50 -080039#define CREATE_TRACE_POINTS
40#include <trace/events/fs_dax.h>
41
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -040042static inline unsigned int pe_order(enum page_entry_size pe_size)
43{
44 if (pe_size == PE_SIZE_PTE)
45 return PAGE_SHIFT - PAGE_SHIFT;
46 if (pe_size == PE_SIZE_PMD)
47 return PMD_SHIFT - PAGE_SHIFT;
48 if (pe_size == PE_SIZE_PUD)
49 return PUD_SHIFT - PAGE_SHIFT;
50 return ~0;
51}
52
Jan Karaac401cc2016-05-12 18:29:18 +020053/* We choose 4096 entries - same as per-zone page wait tables */
54#define DAX_WAIT_TABLE_BITS 12
55#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
56
Ross Zwisler917f3452017-09-06 16:18:58 -070057/* The 'colour' (ie low bits) within a PMD of a page offset. */
58#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
Matthew Wilcox977fbdc2018-01-31 16:17:36 -080059#define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT)
Ross Zwisler917f3452017-09-06 16:18:58 -070060
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -040061/* The order of a PMD entry */
62#define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)
63
Ross Zwislerce95ab02016-11-08 11:31:44 +110064static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
Jan Karaac401cc2016-05-12 18:29:18 +020065
66static int __init init_dax_wait_table(void)
67{
68 int i;
69
70 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
71 init_waitqueue_head(wait_table + i);
72 return 0;
73}
74fs_initcall(init_dax_wait_table);
75
Ross Zwisler527b19d2017-09-06 16:18:51 -070076/*
Matthew Wilcox3159f942017-11-03 13:30:42 -040077 * DAX pagecache entries use XArray value entries so they can't be mistaken
78 * for pages. We use one bit for locking, one bit for the entry size (PMD)
79 * and two more to tell us if the entry is a zero page or an empty entry that
80 * is just used for locking. In total four special bits.
Ross Zwisler527b19d2017-09-06 16:18:51 -070081 *
82 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
83 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
84 * block allocation.
85 */
Matthew Wilcox3159f942017-11-03 13:30:42 -040086#define DAX_SHIFT (4)
87#define DAX_LOCKED (1UL << 0)
88#define DAX_PMD (1UL << 1)
89#define DAX_ZERO_PAGE (1UL << 2)
90#define DAX_EMPTY (1UL << 3)
Ross Zwisler527b19d2017-09-06 16:18:51 -070091
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -040092static unsigned long dax_to_pfn(void *entry)
Ross Zwisler527b19d2017-09-06 16:18:51 -070093{
Matthew Wilcox3159f942017-11-03 13:30:42 -040094 return xa_to_value(entry) >> DAX_SHIFT;
Ross Zwisler527b19d2017-09-06 16:18:51 -070095}
96
Matthew Wilcox9f32d222018-06-12 09:46:30 -040097static void *dax_make_entry(pfn_t pfn, unsigned long flags)
98{
99 return xa_mk_value(flags | (pfn_t_to_pfn(pfn) << DAX_SHIFT));
100}
101
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400102static bool dax_is_locked(void *entry)
103{
104 return xa_to_value(entry) & DAX_LOCKED;
105}
106
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400107static unsigned int dax_entry_order(void *entry)
Ross Zwisler527b19d2017-09-06 16:18:51 -0700108{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400109 if (xa_to_value(entry) & DAX_PMD)
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400110 return PMD_ORDER;
Ross Zwisler527b19d2017-09-06 16:18:51 -0700111 return 0;
112}
113
Matthew Wilcoxfda490d2018-11-16 15:07:31 -0500114static unsigned long dax_is_pmd_entry(void *entry)
Ross Zwisler642261a2016-11-08 11:34:45 +1100115{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400116 return xa_to_value(entry) & DAX_PMD;
Ross Zwisler642261a2016-11-08 11:34:45 +1100117}
118
Matthew Wilcoxfda490d2018-11-16 15:07:31 -0500119static bool dax_is_pte_entry(void *entry)
Ross Zwisler642261a2016-11-08 11:34:45 +1100120{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400121 return !(xa_to_value(entry) & DAX_PMD);
Ross Zwisler642261a2016-11-08 11:34:45 +1100122}
123
124static int dax_is_zero_entry(void *entry)
125{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400126 return xa_to_value(entry) & DAX_ZERO_PAGE;
Ross Zwisler642261a2016-11-08 11:34:45 +1100127}
128
129static int dax_is_empty_entry(void *entry)
130{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400131 return xa_to_value(entry) & DAX_EMPTY;
Ross Zwisler642261a2016-11-08 11:34:45 +1100132}
133
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800134/*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400135 * DAX page cache entry locking
Jan Karaac401cc2016-05-12 18:29:18 +0200136 */
137struct exceptional_entry_key {
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400138 struct xarray *xa;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100139 pgoff_t entry_start;
Jan Karaac401cc2016-05-12 18:29:18 +0200140};
141
142struct wait_exceptional_entry_queue {
Ingo Molnarac6424b2017-06-20 12:06:13 +0200143 wait_queue_entry_t wait;
Jan Karaac401cc2016-05-12 18:29:18 +0200144 struct exceptional_entry_key key;
145};
146
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400147static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas,
148 void *entry, struct exceptional_entry_key *key)
Ross Zwisler63e95b52016-11-08 11:32:20 +1100149{
150 unsigned long hash;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400151 unsigned long index = xas->xa_index;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100152
153 /*
154 * If 'entry' is a PMD, align the 'index' that we use for the wait
155 * queue to the start of that PMD. This ensures that all offsets in
156 * the range covered by the PMD map to the same bit lock.
157 */
Ross Zwisler642261a2016-11-08 11:34:45 +1100158 if (dax_is_pmd_entry(entry))
Ross Zwisler917f3452017-09-06 16:18:58 -0700159 index &= ~PG_PMD_COLOUR;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400160 key->xa = xas->xa;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100161 key->entry_start = index;
162
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400163 hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS);
Ross Zwisler63e95b52016-11-08 11:32:20 +1100164 return wait_table + hash;
165}
166
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400167static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
168 unsigned int mode, int sync, void *keyp)
Jan Karaac401cc2016-05-12 18:29:18 +0200169{
170 struct exceptional_entry_key *key = keyp;
171 struct wait_exceptional_entry_queue *ewait =
172 container_of(wait, struct wait_exceptional_entry_queue, wait);
173
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400174 if (key->xa != ewait->key.xa ||
Ross Zwisler63e95b52016-11-08 11:32:20 +1100175 key->entry_start != ewait->key.entry_start)
Jan Karaac401cc2016-05-12 18:29:18 +0200176 return 0;
177 return autoremove_wake_function(wait, mode, sync, NULL);
178}
179
180/*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700181 * @entry may no longer be the entry at the index in the mapping.
182 * The important information it's conveying is whether the entry at
183 * this index used to be a PMD entry.
Ross Zwislere30331f2017-09-06 16:18:39 -0700184 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400185static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all)
Ross Zwislere30331f2017-09-06 16:18:39 -0700186{
187 struct exceptional_entry_key key;
188 wait_queue_head_t *wq;
189
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400190 wq = dax_entry_waitqueue(xas, entry, &key);
Ross Zwislere30331f2017-09-06 16:18:39 -0700191
192 /*
193 * Checking for locked entry and prepare_to_wait_exclusive() happens
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700194 * under the i_pages lock, ditto for entry handling in our callers.
Ross Zwislere30331f2017-09-06 16:18:39 -0700195 * So at this point all tasks that could have seen our entry locked
196 * must be in the waitqueue and the following check will see them.
197 */
198 if (waitqueue_active(wq))
199 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
200}
201
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400202/*
203 * Look up entry in page cache, wait for it to become unlocked if it
204 * is a DAX entry and return it. The caller must subsequently call
205 * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry()
206 * if it did.
207 *
208 * Must be called with the i_pages lock held.
209 */
210static void *get_unlocked_entry(struct xa_state *xas)
211{
212 void *entry;
213 struct wait_exceptional_entry_queue ewait;
214 wait_queue_head_t *wq;
215
216 init_wait(&ewait.wait);
217 ewait.wait.func = wake_exceptional_entry_func;
218
219 for (;;) {
Matthew Wilcox0e40de02018-11-16 15:19:13 -0500220 entry = xas_find_conflict(xas);
221 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)) ||
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400222 !dax_is_locked(entry))
223 return entry;
224
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400225 wq = dax_entry_waitqueue(xas, entry, &ewait.key);
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400226 prepare_to_wait_exclusive(wq, &ewait.wait,
227 TASK_UNINTERRUPTIBLE);
228 xas_unlock_irq(xas);
229 xas_reset(xas);
230 schedule();
231 finish_wait(wq, &ewait.wait);
232 xas_lock_irq(xas);
233 }
234}
235
Matthew Wilcox55e56f02018-11-27 13:16:34 -0800236/*
237 * The only thing keeping the address space around is the i_pages lock
238 * (it's cycled in clear_inode() after removing the entries from i_pages)
239 * After we call xas_unlock_irq(), we cannot touch xas->xa.
240 */
241static void wait_entry_unlocked(struct xa_state *xas, void *entry)
242{
243 struct wait_exceptional_entry_queue ewait;
244 wait_queue_head_t *wq;
245
246 init_wait(&ewait.wait);
247 ewait.wait.func = wake_exceptional_entry_func;
248
249 wq = dax_entry_waitqueue(xas, entry, &ewait.key);
Dan Williamsd8a70642018-12-21 11:35:53 -0800250 /*
251 * Unlike get_unlocked_entry() there is no guarantee that this
252 * path ever successfully retrieves an unlocked entry before an
253 * inode dies. Perform a non-exclusive wait in case this path
254 * never successfully performs its own wake up.
255 */
256 prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE);
Matthew Wilcox55e56f02018-11-27 13:16:34 -0800257 xas_unlock_irq(xas);
258 schedule();
259 finish_wait(wq, &ewait.wait);
Matthew Wilcox55e56f02018-11-27 13:16:34 -0800260}
261
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400262static void put_unlocked_entry(struct xa_state *xas, void *entry)
263{
264 /* If we were the only waiter woken, wake the next one */
265 if (entry)
266 dax_wake_entry(xas, entry, false);
267}
268
269/*
270 * We used the xa_state to get the entry, but then we locked the entry and
271 * dropped the xa_lock, so we know the xa_state is stale and must be reset
272 * before use.
273 */
274static void dax_unlock_entry(struct xa_state *xas, void *entry)
275{
276 void *old;
277
Matthew Wilcox7ae2ea72018-11-09 20:09:37 -0500278 BUG_ON(dax_is_locked(entry));
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400279 xas_reset(xas);
280 xas_lock_irq(xas);
281 old = xas_store(xas, entry);
282 xas_unlock_irq(xas);
283 BUG_ON(!dax_is_locked(old));
284 dax_wake_entry(xas, entry, false);
285}
286
287/*
288 * Return: The entry stored at this location before it was locked.
289 */
290static void *dax_lock_entry(struct xa_state *xas, void *entry)
291{
292 unsigned long v = xa_to_value(entry);
293 return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
294}
295
Dan Williamsd2c997c2017-12-22 22:02:48 -0800296static unsigned long dax_entry_size(void *entry)
297{
298 if (dax_is_zero_entry(entry))
299 return 0;
300 else if (dax_is_empty_entry(entry))
301 return 0;
302 else if (dax_is_pmd_entry(entry))
303 return PMD_SIZE;
304 else
305 return PAGE_SIZE;
306}
307
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400308static unsigned long dax_end_pfn(void *entry)
Dan Williamsd2c997c2017-12-22 22:02:48 -0800309{
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400310 return dax_to_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800311}
312
313/*
314 * Iterate through all mapped pfns represented by an entry, i.e. skip
315 * 'empty' and 'zero' entries.
316 */
317#define for_each_mapped_pfn(entry, pfn) \
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400318 for (pfn = dax_to_pfn(entry); \
319 pfn < dax_end_pfn(entry); pfn++)
Dan Williamsd2c997c2017-12-22 22:02:48 -0800320
Dan Williams73449da2018-07-13 21:49:50 -0700321/*
322 * TODO: for reflink+dax we need a way to associate a single page with
323 * multiple address_space instances at different linear_page_index()
324 * offsets.
325 */
326static void dax_associate_entry(void *entry, struct address_space *mapping,
327 struct vm_area_struct *vma, unsigned long address)
Dan Williamsd2c997c2017-12-22 22:02:48 -0800328{
Dan Williams73449da2018-07-13 21:49:50 -0700329 unsigned long size = dax_entry_size(entry), pfn, index;
330 int i = 0;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800331
332 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
333 return;
334
Dan Williams73449da2018-07-13 21:49:50 -0700335 index = linear_page_index(vma, address & ~(size - 1));
Dan Williamsd2c997c2017-12-22 22:02:48 -0800336 for_each_mapped_pfn(entry, pfn) {
337 struct page *page = pfn_to_page(pfn);
338
339 WARN_ON_ONCE(page->mapping);
340 page->mapping = mapping;
Dan Williams73449da2018-07-13 21:49:50 -0700341 page->index = index + i++;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800342 }
343}
344
345static void dax_disassociate_entry(void *entry, struct address_space *mapping,
346 bool trunc)
347{
348 unsigned long pfn;
349
350 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
351 return;
352
353 for_each_mapped_pfn(entry, pfn) {
354 struct page *page = pfn_to_page(pfn);
355
356 WARN_ON_ONCE(trunc && page_ref_count(page) > 1);
357 WARN_ON_ONCE(page->mapping && page->mapping != mapping);
358 page->mapping = NULL;
Dan Williams73449da2018-07-13 21:49:50 -0700359 page->index = 0;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800360 }
361}
362
Dan Williams5fac7402018-03-09 17:44:31 -0800363static struct page *dax_busy_page(void *entry)
364{
365 unsigned long pfn;
366
367 for_each_mapped_pfn(entry, pfn) {
368 struct page *page = pfn_to_page(pfn);
369
370 if (page_ref_count(page) > 1)
371 return page;
372 }
373 return NULL;
374}
375
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500376/*
377 * dax_lock_mapping_entry - Lock the DAX entry corresponding to a page
378 * @page: The page whose entry we want to lock
379 *
380 * Context: Process context.
Matthew Wilcox27359fd2018-11-30 11:05:06 -0500381 * Return: A cookie to pass to dax_unlock_page() or 0 if the entry could
382 * not be locked.
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500383 */
Matthew Wilcox27359fd2018-11-30 11:05:06 -0500384dax_entry_t dax_lock_page(struct page *page)
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700385{
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400386 XA_STATE(xas, NULL, 0);
387 void *entry;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700388
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500389 /* Ensure page->mapping isn't freed while we look at it */
390 rcu_read_lock();
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700391 for (;;) {
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400392 struct address_space *mapping = READ_ONCE(page->mapping);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700393
Matthew Wilcox27359fd2018-11-30 11:05:06 -0500394 entry = NULL;
Matthew Wilcoxc93db7b2018-11-27 13:16:33 -0800395 if (!mapping || !dax_mapping(mapping))
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500396 break;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700397
398 /*
399 * In the device-dax case there's no need to lock, a
400 * struct dev_pagemap pin is sufficient to keep the
401 * inode alive, and we assume we have dev_pagemap pin
402 * otherwise we would not have a valid pfn_to_page()
403 * translation.
404 */
Matthew Wilcox27359fd2018-11-30 11:05:06 -0500405 entry = (void *)~0UL;
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400406 if (S_ISCHR(mapping->host->i_mode))
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500407 break;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700408
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400409 xas.xa = &mapping->i_pages;
410 xas_lock_irq(&xas);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700411 if (mapping != page->mapping) {
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400412 xas_unlock_irq(&xas);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700413 continue;
414 }
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400415 xas_set(&xas, page->index);
416 entry = xas_load(&xas);
417 if (dax_is_locked(entry)) {
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500418 rcu_read_unlock();
Matthew Wilcox55e56f02018-11-27 13:16:34 -0800419 wait_entry_unlocked(&xas, entry);
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500420 rcu_read_lock();
Matthew Wilcox6d7cd8c2018-11-06 13:11:57 -0500421 continue;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700422 }
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400423 dax_lock_entry(&xas, entry);
424 xas_unlock_irq(&xas);
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500425 break;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700426 }
Matthew Wilcoxc5bbd452018-11-16 14:37:06 -0500427 rcu_read_unlock();
Matthew Wilcox27359fd2018-11-30 11:05:06 -0500428 return (dax_entry_t)entry;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700429}
430
Matthew Wilcox27359fd2018-11-30 11:05:06 -0500431void dax_unlock_page(struct page *page, dax_entry_t cookie)
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700432{
433 struct address_space *mapping = page->mapping;
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400434 XA_STATE(xas, &mapping->i_pages, page->index);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700435
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400436 if (S_ISCHR(mapping->host->i_mode))
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700437 return;
438
Matthew Wilcox27359fd2018-11-30 11:05:06 -0500439 dax_unlock_entry(&xas, (void *)cookie);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700440}
441
Jan Karaac401cc2016-05-12 18:29:18 +0200442/*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400443 * Find page cache entry at given index. If it is a DAX entry, return it
444 * with the entry locked. If the page cache doesn't contain an entry at
445 * that index, add a locked empty entry.
Jan Karaac401cc2016-05-12 18:29:18 +0200446 *
Matthew Wilcox3159f942017-11-03 13:30:42 -0400447 * When requesting an entry with size DAX_PMD, grab_mapping_entry() will
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400448 * either return that locked entry or will return VM_FAULT_FALLBACK.
449 * This will happen if there are any PTE entries within the PMD range
450 * that we are requesting.
Ross Zwisler642261a2016-11-08 11:34:45 +1100451 *
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400452 * We always favor PTE entries over PMD entries. There isn't a flow where we
453 * evict PTE entries in order to 'upgrade' them to a PMD entry. A PMD
454 * insertion will fail if it finds any PTE entries already in the tree, and a
455 * PTE insertion will cause an existing PMD entry to be unmapped and
456 * downgraded to PTE entries. This happens for both PMD zero pages as
457 * well as PMD empty entries.
Ross Zwisler642261a2016-11-08 11:34:45 +1100458 *
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400459 * The exception to this downgrade path is for PMD entries that have
460 * real storage backing them. We will leave these real PMD entries in
461 * the tree, and PTE writes will simply dirty the entire PMD entry.
Ross Zwisler642261a2016-11-08 11:34:45 +1100462 *
Jan Karaac401cc2016-05-12 18:29:18 +0200463 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
464 * persistent memory the benefit is doubtful. We can add that later if we can
465 * show it helps.
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400466 *
467 * On error, this function does not return an ERR_PTR. Instead it returns
468 * a VM_FAULT code, encoded as an xarray internal entry. The ERR_PTR values
469 * overlap with xarray value entries.
Jan Karaac401cc2016-05-12 18:29:18 +0200470 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400471static void *grab_mapping_entry(struct xa_state *xas,
472 struct address_space *mapping, unsigned long size_flag)
Jan Karaac401cc2016-05-12 18:29:18 +0200473{
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400474 unsigned long index = xas->xa_index;
475 bool pmd_downgrade = false; /* splitting PMD entry into PTE entries? */
476 void *entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200477
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400478retry:
479 xas_lock_irq(xas);
480 entry = get_unlocked_entry(xas);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700481
Ross Zwisler642261a2016-11-08 11:34:45 +1100482 if (entry) {
Matthew Wilcox0e40de02018-11-16 15:19:13 -0500483 if (!xa_is_value(entry)) {
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400484 xas_set_err(xas, EIO);
485 goto out_unlock;
486 }
487
Matthew Wilcox3159f942017-11-03 13:30:42 -0400488 if (size_flag & DAX_PMD) {
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700489 if (dax_is_pte_entry(entry)) {
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400490 put_unlocked_entry(xas, entry);
491 goto fallback;
Ross Zwisler642261a2016-11-08 11:34:45 +1100492 }
493 } else { /* trying to grab a PTE entry */
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700494 if (dax_is_pmd_entry(entry) &&
Ross Zwisler642261a2016-11-08 11:34:45 +1100495 (dax_is_zero_entry(entry) ||
496 dax_is_empty_entry(entry))) {
497 pmd_downgrade = true;
498 }
499 }
500 }
501
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400502 if (pmd_downgrade) {
503 /*
504 * Make sure 'entry' remains valid while we drop
505 * the i_pages lock.
506 */
507 dax_lock_entry(xas, entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200508
Ross Zwisler642261a2016-11-08 11:34:45 +1100509 /*
510 * Besides huge zero pages the only other thing that gets
511 * downgraded are empty entries which don't need to be
512 * unmapped.
513 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400514 if (dax_is_zero_entry(entry)) {
515 xas_unlock_irq(xas);
516 unmap_mapping_pages(mapping,
517 xas->xa_index & ~PG_PMD_COLOUR,
518 PG_PMD_NR, false);
519 xas_reset(xas);
520 xas_lock_irq(xas);
Ross Zwislere11f8b72017-04-07 16:04:57 -0700521 }
522
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400523 dax_disassociate_entry(entry, mapping, false);
524 xas_store(xas, NULL); /* undo the PMD join */
525 dax_wake_entry(xas, entry, true);
526 mapping->nrexceptional--;
527 entry = NULL;
528 xas_set(xas, index);
Jan Karaac401cc2016-05-12 18:29:18 +0200529 }
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400530
531 if (entry) {
532 dax_lock_entry(xas, entry);
533 } else {
534 entry = dax_make_entry(pfn_to_pfn_t(0), size_flag | DAX_EMPTY);
535 dax_lock_entry(xas, entry);
536 if (xas_error(xas))
537 goto out_unlock;
538 mapping->nrexceptional++;
539 }
540
541out_unlock:
542 xas_unlock_irq(xas);
543 if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))
544 goto retry;
545 if (xas->xa_node == XA_ERROR(-ENOMEM))
546 return xa_mk_internal(VM_FAULT_OOM);
547 if (xas_error(xas))
548 return xa_mk_internal(VM_FAULT_SIGBUS);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100549 return entry;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400550fallback:
551 xas_unlock_irq(xas);
552 return xa_mk_internal(VM_FAULT_FALLBACK);
Jan Karaac401cc2016-05-12 18:29:18 +0200553}
554
Dan Williams5fac7402018-03-09 17:44:31 -0800555/**
556 * dax_layout_busy_page - find first pinned page in @mapping
557 * @mapping: address space to scan for a page with ref count > 1
558 *
559 * DAX requires ZONE_DEVICE mapped pages. These pages are never
560 * 'onlined' to the page allocator so they are considered idle when
561 * page->count == 1. A filesystem uses this interface to determine if
562 * any page in the mapping is busy, i.e. for DMA, or other
563 * get_user_pages() usages.
564 *
565 * It is expected that the filesystem is holding locks to block the
566 * establishment of new mappings in this address_space. I.e. it expects
567 * to be able to run unmap_mapping_range() and subsequently not race
568 * mapping_mapped() becoming true.
569 */
570struct page *dax_layout_busy_page(struct address_space *mapping)
571{
Matthew Wilcox084a8992018-05-17 13:03:48 -0400572 XA_STATE(xas, &mapping->i_pages, 0);
573 void *entry;
574 unsigned int scanned = 0;
Dan Williams5fac7402018-03-09 17:44:31 -0800575 struct page *page = NULL;
Dan Williams5fac7402018-03-09 17:44:31 -0800576
577 /*
578 * In the 'limited' case get_user_pages() for dax is disabled.
579 */
580 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
581 return NULL;
582
583 if (!dax_mapping(mapping) || !mapping_mapped(mapping))
584 return NULL;
585
Dan Williams5fac7402018-03-09 17:44:31 -0800586 /*
587 * If we race get_user_pages_fast() here either we'll see the
Matthew Wilcox084a8992018-05-17 13:03:48 -0400588 * elevated page count in the iteration and wait, or
Dan Williams5fac7402018-03-09 17:44:31 -0800589 * get_user_pages_fast() will see that the page it took a reference
590 * against is no longer mapped in the page tables and bail to the
591 * get_user_pages() slow path. The slow path is protected by
592 * pte_lock() and pmd_lock(). New references are not taken without
593 * holding those locks, and unmap_mapping_range() will not zero the
594 * pte or pmd without holding the respective lock, so we are
595 * guaranteed to either see new references or prevent new
596 * references from being established.
597 */
598 unmap_mapping_range(mapping, 0, 0, 1);
599
Matthew Wilcox084a8992018-05-17 13:03:48 -0400600 xas_lock_irq(&xas);
601 xas_for_each(&xas, entry, ULONG_MAX) {
602 if (WARN_ON_ONCE(!xa_is_value(entry)))
603 continue;
604 if (unlikely(dax_is_locked(entry)))
605 entry = get_unlocked_entry(&xas);
606 if (entry)
607 page = dax_busy_page(entry);
608 put_unlocked_entry(&xas, entry);
Dan Williams5fac7402018-03-09 17:44:31 -0800609 if (page)
610 break;
Matthew Wilcox084a8992018-05-17 13:03:48 -0400611 if (++scanned % XA_CHECK_SCHED)
612 continue;
613
614 xas_pause(&xas);
615 xas_unlock_irq(&xas);
616 cond_resched();
617 xas_lock_irq(&xas);
Dan Williams5fac7402018-03-09 17:44:31 -0800618 }
Matthew Wilcox084a8992018-05-17 13:03:48 -0400619 xas_unlock_irq(&xas);
Dan Williams5fac7402018-03-09 17:44:31 -0800620 return page;
621}
622EXPORT_SYMBOL_GPL(dax_layout_busy_page);
623
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400624static int __dax_invalidate_entry(struct address_space *mapping,
Jan Karac6dcf522016-08-10 17:22:44 +0200625 pgoff_t index, bool trunc)
626{
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400627 XA_STATE(xas, &mapping->i_pages, index);
Jan Karac6dcf522016-08-10 17:22:44 +0200628 int ret = 0;
629 void *entry;
Jan Karac6dcf522016-08-10 17:22:44 +0200630
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400631 xas_lock_irq(&xas);
632 entry = get_unlocked_entry(&xas);
Matthew Wilcox3159f942017-11-03 13:30:42 -0400633 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
Jan Karac6dcf522016-08-10 17:22:44 +0200634 goto out;
635 if (!trunc &&
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400636 (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
637 xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
Jan Karac6dcf522016-08-10 17:22:44 +0200638 goto out;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800639 dax_disassociate_entry(entry, mapping, trunc);
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400640 xas_store(&xas, NULL);
Jan Karac6dcf522016-08-10 17:22:44 +0200641 mapping->nrexceptional--;
642 ret = 1;
643out:
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400644 put_unlocked_entry(&xas, entry);
645 xas_unlock_irq(&xas);
Jan Karac6dcf522016-08-10 17:22:44 +0200646 return ret;
647}
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400648
Jan Karaac401cc2016-05-12 18:29:18 +0200649/*
Matthew Wilcox3159f942017-11-03 13:30:42 -0400650 * Delete DAX entry at @index from @mapping. Wait for it
651 * to be unlocked before deleting it.
Jan Karaac401cc2016-05-12 18:29:18 +0200652 */
653int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
654{
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400655 int ret = __dax_invalidate_entry(mapping, index, true);
Jan Karaac401cc2016-05-12 18:29:18 +0200656
Jan Karaac401cc2016-05-12 18:29:18 +0200657 /*
658 * This gets called from truncate / punch_hole path. As such, the caller
659 * must hold locks protecting against concurrent modifications of the
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400660 * page cache (usually fs-private i_mmap_sem for writing). Since the
Matthew Wilcox3159f942017-11-03 13:30:42 -0400661 * caller has seen a DAX entry for this index, we better find it
Jan Karaac401cc2016-05-12 18:29:18 +0200662 * at that index as well...
663 */
Jan Karac6dcf522016-08-10 17:22:44 +0200664 WARN_ON_ONCE(!ret);
665 return ret;
666}
Jan Karaac401cc2016-05-12 18:29:18 +0200667
Jan Karac6dcf522016-08-10 17:22:44 +0200668/*
Matthew Wilcox3159f942017-11-03 13:30:42 -0400669 * Invalidate DAX entry if it is clean.
Jan Karac6dcf522016-08-10 17:22:44 +0200670 */
671int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
672 pgoff_t index)
673{
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400674 return __dax_invalidate_entry(mapping, index, false);
Jan Karaac401cc2016-05-12 18:29:18 +0200675}
676
Dan Williamscccbce62017-01-27 13:31:42 -0800677static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
678 sector_t sector, size_t size, struct page *to,
679 unsigned long vaddr)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800680{
Dan Williamscccbce62017-01-27 13:31:42 -0800681 void *vto, *kaddr;
682 pgoff_t pgoff;
Dan Williamscccbce62017-01-27 13:31:42 -0800683 long rc;
684 int id;
Ross Zwislere2e05392015-08-18 13:55:41 -0600685
Dan Williamscccbce62017-01-27 13:31:42 -0800686 rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
687 if (rc)
688 return rc;
689
690 id = dax_read_lock();
Huaisheng Ye86ed9132018-07-30 15:15:48 +0800691 rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, NULL);
Dan Williamscccbce62017-01-27 13:31:42 -0800692 if (rc < 0) {
693 dax_read_unlock(id);
694 return rc;
695 }
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800696 vto = kmap_atomic(to);
Dan Williamscccbce62017-01-27 13:31:42 -0800697 copy_user_page(vto, (void __force *)kaddr, vaddr, to);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800698 kunmap_atomic(vto);
Dan Williamscccbce62017-01-27 13:31:42 -0800699 dax_read_unlock(id);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800700 return 0;
701}
702
Ross Zwisler642261a2016-11-08 11:34:45 +1100703/*
704 * By this point grab_mapping_entry() has ensured that we have a locked entry
705 * of the appropriate size so we don't have to worry about downgrading PMDs to
706 * PTEs. If we happen to be trying to insert a PTE and there is a PMD
707 * already in the tree, we will skip the insertion and just dirty the PMD as
708 * appropriate.
709 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400710static void *dax_insert_entry(struct xa_state *xas,
711 struct address_space *mapping, struct vm_fault *vmf,
712 void *entry, pfn_t pfn, unsigned long flags, bool dirty)
Ross Zwisler9973c982016-01-22 15:10:47 -0800713{
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400714 void *new_entry = dax_make_entry(pfn, flags);
Ross Zwisler9973c982016-01-22 15:10:47 -0800715
Jan Karaf5b7b742017-11-01 16:36:40 +0100716 if (dirty)
Dmitry Monakhovd2b2a282016-02-05 15:36:55 -0800717 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
Ross Zwisler9973c982016-01-22 15:10:47 -0800718
Matthew Wilcox3159f942017-11-03 13:30:42 -0400719 if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) {
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400720 unsigned long index = xas->xa_index;
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700721 /* we are replacing a zero page with block mapping */
722 if (dax_is_pmd_entry(entry))
Matthew Wilcox977fbdc2018-01-31 16:17:36 -0800723 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400724 PG_PMD_NR, false);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700725 else /* pte entry */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400726 unmap_mapping_pages(mapping, index, 1, false);
Ross Zwisler9973c982016-01-22 15:10:47 -0800727 }
728
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400729 xas_reset(xas);
730 xas_lock_irq(xas);
Dan Williamsd2c997c2017-12-22 22:02:48 -0800731 if (dax_entry_size(entry) != dax_entry_size(new_entry)) {
732 dax_disassociate_entry(entry, mapping, false);
Dan Williams73449da2018-07-13 21:49:50 -0700733 dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
Dan Williamsd2c997c2017-12-22 22:02:48 -0800734 }
Ross Zwisler642261a2016-11-08 11:34:45 +1100735
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700736 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
Ross Zwisler642261a2016-11-08 11:34:45 +1100737 /*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400738 * Only swap our new entry into the page cache if the current
Ross Zwisler642261a2016-11-08 11:34:45 +1100739 * entry is a zero page or an empty entry. If a normal PTE or
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400740 * PMD entry is already in the cache, we leave it alone. This
Ross Zwisler642261a2016-11-08 11:34:45 +1100741 * means that if we are trying to insert a PTE and the
742 * existing entry is a PMD, we will just leave the PMD in the
743 * tree and dirty it if necessary.
744 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400745 void *old = dax_lock_entry(xas, new_entry);
746 WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) |
747 DAX_LOCKED));
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700748 entry = new_entry;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400749 } else {
750 xas_load(xas); /* Walk the xa_state */
Ross Zwisler9973c982016-01-22 15:10:47 -0800751 }
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700752
Jan Karaf5b7b742017-11-01 16:36:40 +0100753 if (dirty)
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400754 xas_set_mark(xas, PAGECACHE_TAG_DIRTY);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700755
Matthew Wilcoxb15cd802018-03-29 22:58:27 -0400756 xas_unlock_irq(xas);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700757 return entry;
Ross Zwisler9973c982016-01-22 15:10:47 -0800758}
759
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400760static inline
761unsigned long pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
Jan Kara4b4bb462016-12-14 15:07:53 -0800762{
763 unsigned long address;
764
765 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
766 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
767 return address;
768}
769
770/* Walk all mappings of a given index of a file and writeprotect them */
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400771static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
772 unsigned long pfn)
Jan Kara4b4bb462016-12-14 15:07:53 -0800773{
774 struct vm_area_struct *vma;
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800775 pte_t pte, *ptep = NULL;
776 pmd_t *pmdp = NULL;
Jan Kara4b4bb462016-12-14 15:07:53 -0800777 spinlock_t *ptl;
Jan Kara4b4bb462016-12-14 15:07:53 -0800778
779 i_mmap_lock_read(mapping);
780 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
Jérôme Glisseac46d4f2018-12-28 00:38:09 -0800781 struct mmu_notifier_range range;
782 unsigned long address;
Jan Kara4b4bb462016-12-14 15:07:53 -0800783
784 cond_resched();
785
786 if (!(vma->vm_flags & VM_SHARED))
787 continue;
788
789 address = pgoff_address(index, vma);
Jérôme Glissea4d1a882017-08-31 17:17:26 -0400790
791 /*
Ira Weiny0cefc362019-01-17 18:20:42 -0500792 * Note because we provide range to follow_pte_pmd it will
Jérôme Glissea4d1a882017-08-31 17:17:26 -0400793 * call mmu_notifier_invalidate_range_start() on our behalf
794 * before taking any lock.
795 */
Jérôme Glisseac46d4f2018-12-28 00:38:09 -0800796 if (follow_pte_pmd(vma->vm_mm, address, &range,
797 &ptep, &pmdp, &ptl))
Jan Kara4b4bb462016-12-14 15:07:53 -0800798 continue;
Jan Kara4b4bb462016-12-14 15:07:53 -0800799
Jérôme Glisse0f108512017-11-15 17:34:07 -0800800 /*
801 * No need to call mmu_notifier_invalidate_range() as we are
802 * downgrading page table protection not changing it to point
803 * to a new page.
804 *
Mike Rapoportad56b732018-03-21 21:22:47 +0200805 * See Documentation/vm/mmu_notifier.rst
Jérôme Glisse0f108512017-11-15 17:34:07 -0800806 */
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800807 if (pmdp) {
808#ifdef CONFIG_FS_DAX_PMD
809 pmd_t pmd;
810
811 if (pfn != pmd_pfn(*pmdp))
812 goto unlock_pmd;
Linus Torvaldsf6f37322017-12-15 18:53:22 -0800813 if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800814 goto unlock_pmd;
815
816 flush_cache_page(vma, address, pfn);
Aneesh Kumar K.V024eee02019-05-13 17:19:11 -0700817 pmd = pmdp_invalidate(vma, address, pmdp);
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800818 pmd = pmd_wrprotect(pmd);
819 pmd = pmd_mkclean(pmd);
820 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800821unlock_pmd:
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800822#endif
Jan H. Schönherree190ca2018-01-31 16:14:04 -0800823 spin_unlock(ptl);
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800824 } else {
825 if (pfn != pte_pfn(*ptep))
826 goto unlock_pte;
827 if (!pte_dirty(*ptep) && !pte_write(*ptep))
828 goto unlock_pte;
829
830 flush_cache_page(vma, address, pfn);
831 pte = ptep_clear_flush(vma, address, ptep);
832 pte = pte_wrprotect(pte);
833 pte = pte_mkclean(pte);
834 set_pte_at(vma->vm_mm, address, ptep, pte);
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800835unlock_pte:
836 pte_unmap_unlock(ptep, ptl);
837 }
Jan Kara4b4bb462016-12-14 15:07:53 -0800838
Jérôme Glisseac46d4f2018-12-28 00:38:09 -0800839 mmu_notifier_invalidate_range_end(&range);
Jan Kara4b4bb462016-12-14 15:07:53 -0800840 }
841 i_mmap_unlock_read(mapping);
842}
843
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400844static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
845 struct address_space *mapping, void *entry)
Ross Zwisler9973c982016-01-22 15:10:47 -0800846{
Matthew Wilcoxe4b34482019-03-01 11:12:41 -0800847 unsigned long pfn, index, count;
Dan Williams3fe07912017-10-14 17:13:45 -0700848 long ret = 0;
Ross Zwisler9973c982016-01-22 15:10:47 -0800849
Ross Zwisler9973c982016-01-22 15:10:47 -0800850 /*
Jan Karaa6abc2c2016-12-14 15:07:47 -0800851 * A page got tagged dirty in DAX mapping? Something is seriously
852 * wrong.
Ross Zwisler9973c982016-01-22 15:10:47 -0800853 */
Matthew Wilcox3159f942017-11-03 13:30:42 -0400854 if (WARN_ON(!xa_is_value(entry)))
Jan Karaa6abc2c2016-12-14 15:07:47 -0800855 return -EIO;
Ross Zwisler9973c982016-01-22 15:10:47 -0800856
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400857 if (unlikely(dax_is_locked(entry))) {
858 void *old_entry = entry;
859
860 entry = get_unlocked_entry(xas);
861
862 /* Entry got punched out / reallocated? */
863 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
864 goto put_unlocked;
865 /*
866 * Entry got reallocated elsewhere? No need to writeback.
867 * We have to compare pfns as we must not bail out due to
868 * difference in lockbit or entry type.
869 */
870 if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
871 goto put_unlocked;
872 if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
873 dax_is_zero_entry(entry))) {
874 ret = -EIO;
875 goto put_unlocked;
876 }
877
878 /* Another fsync thread may have already done this entry */
879 if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
880 goto put_unlocked;
Ross Zwisler9973c982016-01-22 15:10:47 -0800881 }
882
Jan Karaa6abc2c2016-12-14 15:07:47 -0800883 /* Lock the entry to serialize with page faults */
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400884 dax_lock_entry(xas, entry);
885
Jan Karaa6abc2c2016-12-14 15:07:47 -0800886 /*
887 * We can clear the tag now but we have to be careful so that concurrent
888 * dax_writeback_one() calls for the same index cannot finish before we
889 * actually flush the caches. This is achieved as the calls will look
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700890 * at the entry only under the i_pages lock and once they do that
891 * they will see the entry locked and wait for it to unlock.
Jan Karaa6abc2c2016-12-14 15:07:47 -0800892 */
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400893 xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
894 xas_unlock_irq(xas);
Jan Karaa6abc2c2016-12-14 15:07:47 -0800895
Ross Zwisler642261a2016-11-08 11:34:45 +1100896 /*
Matthew Wilcoxe4b34482019-03-01 11:12:41 -0800897 * If dax_writeback_mapping_range() was given a wbc->range_start
898 * in the middle of a PMD, the 'index' we use needs to be
899 * aligned to the start of the PMD.
Dan Williams3fe07912017-10-14 17:13:45 -0700900 * This allows us to flush for PMD_SIZE and not have to worry about
901 * partial PMD writebacks.
Ross Zwisler642261a2016-11-08 11:34:45 +1100902 */
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400903 pfn = dax_to_pfn(entry);
Matthew Wilcoxe4b34482019-03-01 11:12:41 -0800904 count = 1UL << dax_entry_order(entry);
905 index = xas->xa_index & ~(count - 1);
Dan Williamscccbce62017-01-27 13:31:42 -0800906
Matthew Wilcoxe4b34482019-03-01 11:12:41 -0800907 dax_entry_mkclean(mapping, index, pfn);
908 dax_flush(dax_dev, page_address(pfn_to_page(pfn)), count * PAGE_SIZE);
Jan Kara4b4bb462016-12-14 15:07:53 -0800909 /*
910 * After we have flushed the cache, we can clear the dirty tag. There
911 * cannot be new dirty data in the pfn after the flush has completed as
912 * the pfn mappings are writeprotected and fault waits for mapping
913 * entry lock.
914 */
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400915 xas_reset(xas);
916 xas_lock_irq(xas);
917 xas_store(xas, entry);
918 xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
919 dax_wake_entry(xas, entry, false);
920
Matthew Wilcoxe4b34482019-03-01 11:12:41 -0800921 trace_dax_writeback_one(mapping->host, index, count);
Ross Zwisler9973c982016-01-22 15:10:47 -0800922 return ret;
923
Jan Karaa6abc2c2016-12-14 15:07:47 -0800924 put_unlocked:
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400925 put_unlocked_entry(xas, entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800926 return ret;
927}
928
929/*
930 * Flush the mapping to the persistent domain within the byte range of [start,
931 * end]. This is required by data integrity operations to ensure file data is
932 * on persistent storage prior to completion of the operation.
933 */
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800934int dax_writeback_mapping_range(struct address_space *mapping,
935 struct block_device *bdev, struct writeback_control *wbc)
Ross Zwisler9973c982016-01-22 15:10:47 -0800936{
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400937 XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
Ross Zwisler9973c982016-01-22 15:10:47 -0800938 struct inode *inode = mapping->host;
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400939 pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
Dan Williamscccbce62017-01-27 13:31:42 -0800940 struct dax_device *dax_dev;
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400941 void *entry;
942 int ret = 0;
943 unsigned int scanned = 0;
Ross Zwisler9973c982016-01-22 15:10:47 -0800944
945 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
946 return -EIO;
947
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800948 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
949 return 0;
950
Dan Williamscccbce62017-01-27 13:31:42 -0800951 dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
952 if (!dax_dev)
953 return -EIO;
954
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400955 trace_dax_writeback_range(inode, xas.xa_index, end_index);
Ross Zwisler9973c982016-01-22 15:10:47 -0800956
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400957 tag_pages_for_writeback(mapping, xas.xa_index, end_index);
Ross Zwislerd14a3f42017-05-08 16:00:10 -0700958
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400959 xas_lock_irq(&xas);
960 xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
961 ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
962 if (ret < 0) {
963 mapping_set_error(mapping, ret);
Ross Zwisler9973c982016-01-22 15:10:47 -0800964 break;
Ross Zwisler9973c982016-01-22 15:10:47 -0800965 }
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400966 if (++scanned % XA_CHECK_SCHED)
967 continue;
968
969 xas_pause(&xas);
970 xas_unlock_irq(&xas);
971 cond_resched();
972 xas_lock_irq(&xas);
Ross Zwisler9973c982016-01-22 15:10:47 -0800973 }
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400974 xas_unlock_irq(&xas);
Dan Williamscccbce62017-01-27 13:31:42 -0800975 put_dax(dax_dev);
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400976 trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
977 return ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800978}
979EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
980
Jan Kara31a6f1a2017-11-01 16:36:32 +0100981static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800982{
Linus Torvaldsa3841f92017-11-17 09:51:57 -0800983 return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
Jan Kara31a6f1a2017-11-01 16:36:32 +0100984}
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800985
Jan Kara5e161e42017-11-01 16:36:33 +0100986static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
987 pfn_t *pfnp)
988{
989 const sector_t sector = dax_iomap_sector(iomap, pos);
990 pgoff_t pgoff;
Jan Kara5e161e42017-11-01 16:36:33 +0100991 int id, rc;
992 long length;
993
994 rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
Dan Williamscccbce62017-01-27 13:31:42 -0800995 if (rc)
996 return rc;
Dan Williamscccbce62017-01-27 13:31:42 -0800997 id = dax_read_lock();
Jan Kara5e161e42017-11-01 16:36:33 +0100998 length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
Huaisheng Ye86ed9132018-07-30 15:15:48 +0800999 NULL, pfnp);
Jan Kara5e161e42017-11-01 16:36:33 +01001000 if (length < 0) {
1001 rc = length;
1002 goto out;
Dan Williamscccbce62017-01-27 13:31:42 -08001003 }
Jan Kara5e161e42017-11-01 16:36:33 +01001004 rc = -EINVAL;
1005 if (PFN_PHYS(length) < size)
1006 goto out;
1007 if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
1008 goto out;
1009 /* For larger pages we need devmap */
1010 if (length > 1 && !pfn_t_devmap(*pfnp))
1011 goto out;
1012 rc = 0;
1013out:
Dan Williamscccbce62017-01-27 13:31:42 -08001014 dax_read_unlock(id);
Jan Kara5e161e42017-11-01 16:36:33 +01001015 return rc;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -08001016}
1017
Ross Zwislere30331f2017-09-06 16:18:39 -07001018/*
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001019 * The user has performed a load from a hole in the file. Allocating a new
1020 * page in the file would cause excessive storage usage for workloads with
1021 * sparse files. Instead we insert a read-only mapping of the 4k zero page.
1022 * If this page is ever written to we will re-fault and change the mapping to
1023 * point to real DAX storage instead.
Ross Zwislere30331f2017-09-06 16:18:39 -07001024 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001025static vm_fault_t dax_load_hole(struct xa_state *xas,
1026 struct address_space *mapping, void **entry,
1027 struct vm_fault *vmf)
Ross Zwislere30331f2017-09-06 16:18:39 -07001028{
1029 struct inode *inode = mapping->host;
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001030 unsigned long vaddr = vmf->address;
Matthew Wilcoxb90ca5c2018-09-11 21:27:44 -07001031 pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr));
1032 vm_fault_t ret;
Ross Zwislere30331f2017-09-06 16:18:39 -07001033
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001034 *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
Matthew Wilcox3159f942017-11-03 13:30:42 -04001035 DAX_ZERO_PAGE, false);
1036
Souptick Joarderab77dab2018-06-07 17:04:29 -07001037 ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
Ross Zwislere30331f2017-09-06 16:18:39 -07001038 trace_dax_load_hole(inode, vmf, ret);
1039 return ret;
1040}
1041
Vishal Verma4b0228f2016-04-21 15:13:46 -04001042static bool dax_range_is_aligned(struct block_device *bdev,
1043 unsigned int offset, unsigned int length)
1044{
1045 unsigned short sector_size = bdev_logical_block_size(bdev);
1046
1047 if (!IS_ALIGNED(offset, sector_size))
1048 return false;
1049 if (!IS_ALIGNED(length, sector_size))
1050 return false;
1051
1052 return true;
1053}
1054
Dan Williamscccbce62017-01-27 13:31:42 -08001055int __dax_zero_page_range(struct block_device *bdev,
1056 struct dax_device *dax_dev, sector_t sector,
1057 unsigned int offset, unsigned int size)
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001058{
Dan Williamscccbce62017-01-27 13:31:42 -08001059 if (dax_range_is_aligned(bdev, offset, size)) {
1060 sector_t start_sector = sector + (offset >> 9);
Vishal Verma4b0228f2016-04-21 15:13:46 -04001061
1062 return blkdev_issue_zeroout(bdev, start_sector,
Linus Torvalds53ef7d02017-05-05 18:49:20 -07001063 size >> 9, GFP_NOFS, 0);
Vishal Verma4b0228f2016-04-21 15:13:46 -04001064 } else {
Dan Williamscccbce62017-01-27 13:31:42 -08001065 pgoff_t pgoff;
1066 long rc, id;
1067 void *kaddr;
Dan Williamscccbce62017-01-27 13:31:42 -08001068
Dan Williamse84b83b2017-05-10 19:38:13 -07001069 rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
Dan Williamscccbce62017-01-27 13:31:42 -08001070 if (rc)
1071 return rc;
1072
1073 id = dax_read_lock();
Huaisheng Ye86ed9132018-07-30 15:15:48 +08001074 rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL);
Dan Williamscccbce62017-01-27 13:31:42 -08001075 if (rc < 0) {
1076 dax_read_unlock(id);
1077 return rc;
1078 }
Dan Williams81f55872017-05-29 13:12:20 -07001079 memset(kaddr + offset, 0, size);
Mikulas Patockac3ca0152017-08-31 21:47:43 -04001080 dax_flush(dax_dev, kaddr + offset, size);
Dan Williamscccbce62017-01-27 13:31:42 -08001081 dax_read_unlock(id);
Vishal Verma4b0228f2016-04-21 15:13:46 -04001082 }
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001083 return 0;
1084}
1085EXPORT_SYMBOL_GPL(__dax_zero_page_range);
1086
Christoph Hellwiga254e562016-09-19 11:24:49 +10001087static loff_t
Ross Zwisler11c59c92016-11-08 11:32:46 +11001088dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
Christoph Hellwiga254e562016-09-19 11:24:49 +10001089 struct iomap *iomap)
1090{
Dan Williamscccbce62017-01-27 13:31:42 -08001091 struct block_device *bdev = iomap->bdev;
1092 struct dax_device *dax_dev = iomap->dax_dev;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001093 struct iov_iter *iter = data;
1094 loff_t end = pos + length, done = 0;
1095 ssize_t ret = 0;
Dan Williamsa77d4782018-03-16 17:36:44 -07001096 size_t xfer;
Dan Williamscccbce62017-01-27 13:31:42 -08001097 int id;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001098
1099 if (iov_iter_rw(iter) == READ) {
1100 end = min(end, i_size_read(inode));
1101 if (pos >= end)
1102 return 0;
1103
1104 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
1105 return iov_iter_zero(min(length, end - pos), iter);
1106 }
1107
1108 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
1109 return -EIO;
1110
Jan Karae3fce682016-08-10 17:10:28 +02001111 /*
1112 * Write can allocate block for an area which has a hole page mapped
1113 * into page tables. We have to tear down these mappings so that data
1114 * written by write(2) is visible in mmap.
1115 */
Jan Karacd656372017-05-12 15:46:50 -07001116 if (iomap->flags & IOMAP_F_NEW) {
Jan Karae3fce682016-08-10 17:10:28 +02001117 invalidate_inode_pages2_range(inode->i_mapping,
1118 pos >> PAGE_SHIFT,
1119 (end - 1) >> PAGE_SHIFT);
1120 }
1121
Dan Williamscccbce62017-01-27 13:31:42 -08001122 id = dax_read_lock();
Christoph Hellwiga254e562016-09-19 11:24:49 +10001123 while (pos < end) {
1124 unsigned offset = pos & (PAGE_SIZE - 1);
Dan Williamscccbce62017-01-27 13:31:42 -08001125 const size_t size = ALIGN(length + offset, PAGE_SIZE);
1126 const sector_t sector = dax_iomap_sector(iomap, pos);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001127 ssize_t map_len;
Dan Williamscccbce62017-01-27 13:31:42 -08001128 pgoff_t pgoff;
1129 void *kaddr;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001130
Michal Hockod1908f52017-02-03 13:13:26 -08001131 if (fatal_signal_pending(current)) {
1132 ret = -EINTR;
1133 break;
1134 }
1135
Dan Williamscccbce62017-01-27 13:31:42 -08001136 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
1137 if (ret)
1138 break;
1139
1140 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
Huaisheng Ye86ed9132018-07-30 15:15:48 +08001141 &kaddr, NULL);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001142 if (map_len < 0) {
1143 ret = map_len;
1144 break;
1145 }
1146
Dan Williamscccbce62017-01-27 13:31:42 -08001147 map_len = PFN_PHYS(map_len);
1148 kaddr += offset;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001149 map_len -= offset;
1150 if (map_len > end - pos)
1151 map_len = end - pos;
1152
Ross Zwislera2e050f2017-09-06 16:18:54 -07001153 /*
1154 * The userspace address for the memory copy has already been
1155 * validated via access_ok() in either vfs_read() or
1156 * vfs_write(), depending on which operation we are doing.
1157 */
Christoph Hellwiga254e562016-09-19 11:24:49 +10001158 if (iov_iter_rw(iter) == WRITE)
Dan Williamsa77d4782018-03-16 17:36:44 -07001159 xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
Dan Williamsfec53772017-05-29 21:56:49 -07001160 map_len, iter);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001161 else
Dan Williamsa77d4782018-03-16 17:36:44 -07001162 xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
Dan Williamsb3a9a0c2018-05-02 06:46:33 -07001163 map_len, iter);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001164
Dan Williamsa77d4782018-03-16 17:36:44 -07001165 pos += xfer;
1166 length -= xfer;
1167 done += xfer;
1168
1169 if (xfer == 0)
1170 ret = -EFAULT;
1171 if (xfer < map_len)
1172 break;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001173 }
Dan Williamscccbce62017-01-27 13:31:42 -08001174 dax_read_unlock(id);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001175
1176 return done ? done : ret;
1177}
1178
1179/**
Ross Zwisler11c59c92016-11-08 11:32:46 +11001180 * dax_iomap_rw - Perform I/O to a DAX file
Christoph Hellwiga254e562016-09-19 11:24:49 +10001181 * @iocb: The control block for this I/O
1182 * @iter: The addresses to do I/O from or to
1183 * @ops: iomap ops passed from the file system
1184 *
1185 * This function performs read and write operations to directly mapped
1186 * persistent memory. The callers needs to take care of read/write exclusion
1187 * and evicting any page cache pages in the region under I/O.
1188 */
1189ssize_t
Ross Zwisler11c59c92016-11-08 11:32:46 +11001190dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
Christoph Hellwig8ff6daa2017-01-27 23:20:26 -08001191 const struct iomap_ops *ops)
Christoph Hellwiga254e562016-09-19 11:24:49 +10001192{
1193 struct address_space *mapping = iocb->ki_filp->f_mapping;
1194 struct inode *inode = mapping->host;
1195 loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1196 unsigned flags = 0;
1197
Christoph Hellwig168316d2017-02-08 14:43:13 -05001198 if (iov_iter_rw(iter) == WRITE) {
1199 lockdep_assert_held_exclusive(&inode->i_rwsem);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001200 flags |= IOMAP_WRITE;
Christoph Hellwig168316d2017-02-08 14:43:13 -05001201 } else {
1202 lockdep_assert_held(&inode->i_rwsem);
1203 }
Christoph Hellwiga254e562016-09-19 11:24:49 +10001204
Christoph Hellwiga254e562016-09-19 11:24:49 +10001205 while (iov_iter_count(iter)) {
1206 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
Ross Zwisler11c59c92016-11-08 11:32:46 +11001207 iter, dax_iomap_actor);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001208 if (ret <= 0)
1209 break;
1210 pos += ret;
1211 done += ret;
1212 }
1213
1214 iocb->ki_pos += done;
1215 return done ? done : ret;
1216}
Ross Zwisler11c59c92016-11-08 11:32:46 +11001217EXPORT_SYMBOL_GPL(dax_iomap_rw);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001218
Souptick Joarderab77dab2018-06-07 17:04:29 -07001219static vm_fault_t dax_fault_return(int error)
Jan Kara9f141d62016-10-19 14:34:31 +02001220{
1221 if (error == 0)
1222 return VM_FAULT_NOPAGE;
Souptick Joarderc9aed742019-01-05 00:54:11 +05301223 return vmf_error(error);
Jan Kara9f141d62016-10-19 14:34:31 +02001224}
1225
Dan Williamsaaa422c2017-11-13 16:38:44 -08001226/*
1227 * MAP_SYNC on a dax mapping guarantees dirty metadata is
1228 * flushed on write-faults (non-cow), but not read-faults.
1229 */
1230static bool dax_fault_is_synchronous(unsigned long flags,
1231 struct vm_area_struct *vma, struct iomap *iomap)
1232{
1233 return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
1234 && (iomap->flags & IOMAP_F_DIRTY);
1235}
1236
Souptick Joarderab77dab2018-06-07 17:04:29 -07001237static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
Jan Karac0b24622018-01-07 16:38:43 -05001238 int *iomap_errp, const struct iomap_ops *ops)
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001239{
Jan Karaa0987ad2017-11-01 16:36:34 +01001240 struct vm_area_struct *vma = vmf->vma;
1241 struct address_space *mapping = vma->vm_file->f_mapping;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001242 XA_STATE(xas, &mapping->i_pages, vmf->pgoff);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001243 struct inode *inode = mapping->host;
Jan Kara1a29d852016-12-14 15:07:01 -08001244 unsigned long vaddr = vmf->address;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001245 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001246 struct iomap iomap = { 0 };
Jan Kara9484ab12016-11-10 10:26:50 +11001247 unsigned flags = IOMAP_FAULT;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001248 int error, major = 0;
Jan Karad2c43ef2017-11-01 16:36:35 +01001249 bool write = vmf->flags & FAULT_FLAG_WRITE;
Jan Karacaa51d22017-11-01 16:36:42 +01001250 bool sync;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001251 vm_fault_t ret = 0;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001252 void *entry;
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001253 pfn_t pfn;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001254
Souptick Joarderab77dab2018-06-07 17:04:29 -07001255 trace_dax_pte_fault(inode, vmf, ret);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001256 /*
1257 * Check whether offset isn't beyond end of file now. Caller is supposed
1258 * to hold locks serializing us with truncate / punch hole so this is
1259 * a reliable test.
1260 */
Ross Zwislera9c42b32017-05-08 16:00:00 -07001261 if (pos >= i_size_read(inode)) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001262 ret = VM_FAULT_SIGBUS;
Ross Zwislera9c42b32017-05-08 16:00:00 -07001263 goto out;
1264 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001265
Jan Karad2c43ef2017-11-01 16:36:35 +01001266 if (write && !vmf->cow_page)
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001267 flags |= IOMAP_WRITE;
1268
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001269 entry = grab_mapping_entry(&xas, mapping, 0);
1270 if (xa_is_internal(entry)) {
1271 ret = xa_to_internal(entry);
Jan Kara13e451f2017-05-12 15:46:57 -07001272 goto out;
1273 }
1274
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001275 /*
Ross Zwislere2093922017-06-02 14:46:37 -07001276 * It is possible, particularly with mixed reads & writes to private
1277 * mappings, that we have raced with a PMD fault that overlaps with
1278 * the PTE we need to set up. If so just return and the fault will be
1279 * retried.
1280 */
1281 if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001282 ret = VM_FAULT_NOPAGE;
Ross Zwislere2093922017-06-02 14:46:37 -07001283 goto unlock_entry;
1284 }
1285
1286 /*
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001287 * Note that we don't bother to use iomap_apply here: DAX required
1288 * the file system block size to be equal the page size, which means
1289 * that we never have to deal with more than a single extent here.
1290 */
1291 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
Jan Karac0b24622018-01-07 16:38:43 -05001292 if (iomap_errp)
1293 *iomap_errp = error;
Ross Zwislera9c42b32017-05-08 16:00:00 -07001294 if (error) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001295 ret = dax_fault_return(error);
Jan Kara13e451f2017-05-12 15:46:57 -07001296 goto unlock_entry;
Ross Zwislera9c42b32017-05-08 16:00:00 -07001297 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001298 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
Jan Kara13e451f2017-05-12 15:46:57 -07001299 error = -EIO; /* fs corruption? */
1300 goto error_finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001301 }
1302
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001303 if (vmf->cow_page) {
Jan Kara31a6f1a2017-11-01 16:36:32 +01001304 sector_t sector = dax_iomap_sector(&iomap, pos);
1305
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001306 switch (iomap.type) {
1307 case IOMAP_HOLE:
1308 case IOMAP_UNWRITTEN:
1309 clear_user_highpage(vmf->cow_page, vaddr);
1310 break;
1311 case IOMAP_MAPPED:
Dan Williamscccbce62017-01-27 13:31:42 -08001312 error = copy_user_dax(iomap.bdev, iomap.dax_dev,
1313 sector, PAGE_SIZE, vmf->cow_page, vaddr);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001314 break;
1315 default:
1316 WARN_ON_ONCE(1);
1317 error = -EIO;
1318 break;
1319 }
1320
1321 if (error)
Jan Kara13e451f2017-05-12 15:46:57 -07001322 goto error_finish_iomap;
Jan Karab1aa8122016-12-14 15:07:24 -08001323
1324 __SetPageUptodate(vmf->cow_page);
Souptick Joarderab77dab2018-06-07 17:04:29 -07001325 ret = finish_fault(vmf);
1326 if (!ret)
1327 ret = VM_FAULT_DONE_COW;
Jan Kara13e451f2017-05-12 15:46:57 -07001328 goto finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001329 }
1330
Dan Williamsaaa422c2017-11-13 16:38:44 -08001331 sync = dax_fault_is_synchronous(flags, vma, &iomap);
Jan Karacaa51d22017-11-01 16:36:42 +01001332
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001333 switch (iomap.type) {
1334 case IOMAP_MAPPED:
1335 if (iomap.flags & IOMAP_F_NEW) {
1336 count_vm_event(PGMAJFAULT);
Jan Karaa0987ad2017-11-01 16:36:34 +01001337 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001338 major = VM_FAULT_MAJOR;
1339 }
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001340 error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
1341 if (error < 0)
1342 goto error_finish_iomap;
1343
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001344 entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
Jan Karacaa51d22017-11-01 16:36:42 +01001345 0, write && !sync);
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001346
Jan Karacaa51d22017-11-01 16:36:42 +01001347 /*
1348 * If we are doing synchronous page fault and inode needs fsync,
1349 * we can insert PTE into page tables only after that happens.
1350 * Skip insertion for now and return the pfn so that caller can
1351 * insert it after fsync is done.
1352 */
1353 if (sync) {
1354 if (WARN_ON_ONCE(!pfnp)) {
1355 error = -EIO;
1356 goto error_finish_iomap;
1357 }
1358 *pfnp = pfn;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001359 ret = VM_FAULT_NEEDDSYNC | major;
Jan Karacaa51d22017-11-01 16:36:42 +01001360 goto finish_iomap;
1361 }
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001362 trace_dax_insert_mapping(inode, vmf, entry);
1363 if (write)
Souptick Joarderab77dab2018-06-07 17:04:29 -07001364 ret = vmf_insert_mixed_mkwrite(vma, vaddr, pfn);
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001365 else
Souptick Joarderab77dab2018-06-07 17:04:29 -07001366 ret = vmf_insert_mixed(vma, vaddr, pfn);
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001367
Souptick Joarderab77dab2018-06-07 17:04:29 -07001368 goto finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001369 case IOMAP_UNWRITTEN:
1370 case IOMAP_HOLE:
Jan Karad2c43ef2017-11-01 16:36:35 +01001371 if (!write) {
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001372 ret = dax_load_hole(&xas, mapping, &entry, vmf);
Jan Kara13e451f2017-05-12 15:46:57 -07001373 goto finish_iomap;
Ross Zwisler15502902016-11-08 11:33:26 +11001374 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001375 /*FALLTHRU*/
1376 default:
1377 WARN_ON_ONCE(1);
1378 error = -EIO;
1379 break;
1380 }
1381
Jan Kara13e451f2017-05-12 15:46:57 -07001382 error_finish_iomap:
Souptick Joarderab77dab2018-06-07 17:04:29 -07001383 ret = dax_fault_return(error);
Jan Kara9f141d62016-10-19 14:34:31 +02001384 finish_iomap:
1385 if (ops->iomap_end) {
1386 int copied = PAGE_SIZE;
1387
Souptick Joarderab77dab2018-06-07 17:04:29 -07001388 if (ret & VM_FAULT_ERROR)
Jan Kara9f141d62016-10-19 14:34:31 +02001389 copied = 0;
1390 /*
1391 * The fault is done by now and there's no way back (other
1392 * thread may be already happily using PTE we have installed).
1393 * Just ignore error from ->iomap_end since we cannot do much
1394 * with it.
1395 */
1396 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
Ross Zwisler15502902016-11-08 11:33:26 +11001397 }
Jan Kara13e451f2017-05-12 15:46:57 -07001398 unlock_entry:
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001399 dax_unlock_entry(&xas, entry);
Jan Kara13e451f2017-05-12 15:46:57 -07001400 out:
Souptick Joarderab77dab2018-06-07 17:04:29 -07001401 trace_dax_pte_fault_done(inode, vmf, ret);
1402 return ret | major;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001403}
Ross Zwisler642261a2016-11-08 11:34:45 +11001404
1405#ifdef CONFIG_FS_DAX_PMD
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001406static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,
1407 struct iomap *iomap, void **entry)
Ross Zwisler642261a2016-11-08 11:34:45 +11001408{
Dave Jiangf4200392017-02-22 15:40:06 -08001409 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1410 unsigned long pmd_addr = vmf->address & PMD_MASK;
Aneesh Kumar K.V11cf9d82019-03-09 17:37:21 +05301411 struct vm_area_struct *vma = vmf->vma;
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001412 struct inode *inode = mapping->host;
Aneesh Kumar K.V11cf9d82019-03-09 17:37:21 +05301413 pgtable_t pgtable = NULL;
Ross Zwisler642261a2016-11-08 11:34:45 +11001414 struct page *zero_page;
1415 spinlock_t *ptl;
1416 pmd_t pmd_entry;
Dan Williams3fe07912017-10-14 17:13:45 -07001417 pfn_t pfn;
Ross Zwisler642261a2016-11-08 11:34:45 +11001418
Dave Jiangf4200392017-02-22 15:40:06 -08001419 zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
Ross Zwisler642261a2016-11-08 11:34:45 +11001420
1421 if (unlikely(!zero_page))
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001422 goto fallback;
Ross Zwisler642261a2016-11-08 11:34:45 +11001423
Dan Williams3fe07912017-10-14 17:13:45 -07001424 pfn = page_to_pfn_t(zero_page);
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001425 *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
Matthew Wilcox3159f942017-11-03 13:30:42 -04001426 DAX_PMD | DAX_ZERO_PAGE, false);
Ross Zwisler642261a2016-11-08 11:34:45 +11001427
Aneesh Kumar K.V11cf9d82019-03-09 17:37:21 +05301428 if (arch_needs_pgtable_deposit()) {
1429 pgtable = pte_alloc_one(vma->vm_mm);
1430 if (!pgtable)
1431 return VM_FAULT_OOM;
1432 }
1433
Dave Jiangf4200392017-02-22 15:40:06 -08001434 ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1435 if (!pmd_none(*(vmf->pmd))) {
Ross Zwisler642261a2016-11-08 11:34:45 +11001436 spin_unlock(ptl);
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001437 goto fallback;
Ross Zwisler642261a2016-11-08 11:34:45 +11001438 }
1439
Aneesh Kumar K.V11cf9d82019-03-09 17:37:21 +05301440 if (pgtable) {
1441 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
1442 mm_inc_nr_ptes(vma->vm_mm);
1443 }
Dave Jiangf4200392017-02-22 15:40:06 -08001444 pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
Ross Zwisler642261a2016-11-08 11:34:45 +11001445 pmd_entry = pmd_mkhuge(pmd_entry);
Dave Jiangf4200392017-02-22 15:40:06 -08001446 set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
Ross Zwisler642261a2016-11-08 11:34:45 +11001447 spin_unlock(ptl);
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001448 trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);
Ross Zwisler642261a2016-11-08 11:34:45 +11001449 return VM_FAULT_NOPAGE;
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001450
1451fallback:
Aneesh Kumar K.V11cf9d82019-03-09 17:37:21 +05301452 if (pgtable)
1453 pte_free(vma->vm_mm, pgtable);
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001454 trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001455 return VM_FAULT_FALLBACK;
Ross Zwisler642261a2016-11-08 11:34:45 +11001456}
1457
Souptick Joarderab77dab2018-06-07 17:04:29 -07001458static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
Dave Jianga2d58162017-02-24 14:56:59 -08001459 const struct iomap_ops *ops)
Ross Zwisler642261a2016-11-08 11:34:45 +11001460{
Dave Jiangf4200392017-02-22 15:40:06 -08001461 struct vm_area_struct *vma = vmf->vma;
Ross Zwisler642261a2016-11-08 11:34:45 +11001462 struct address_space *mapping = vma->vm_file->f_mapping;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001463 XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);
Dave Jiangd8a849e2017-02-22 15:40:03 -08001464 unsigned long pmd_addr = vmf->address & PMD_MASK;
1465 bool write = vmf->flags & FAULT_FLAG_WRITE;
Jan Karacaa51d22017-11-01 16:36:42 +01001466 bool sync;
Jan Kara9484ab12016-11-10 10:26:50 +11001467 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
Ross Zwisler642261a2016-11-08 11:34:45 +11001468 struct inode *inode = mapping->host;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001469 vm_fault_t result = VM_FAULT_FALLBACK;
Ross Zwisler642261a2016-11-08 11:34:45 +11001470 struct iomap iomap = { 0 };
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001471 pgoff_t max_pgoff;
Ross Zwisler642261a2016-11-08 11:34:45 +11001472 void *entry;
1473 loff_t pos;
1474 int error;
Jan Kara302a5e32017-11-01 16:36:37 +01001475 pfn_t pfn;
Ross Zwisler642261a2016-11-08 11:34:45 +11001476
Ross Zwisler282a8e02017-02-22 15:39:50 -08001477 /*
1478 * Check whether offset isn't beyond end of file now. Caller is
1479 * supposed to hold locks serializing us with truncate / punch hole so
1480 * this is a reliable test.
1481 */
Jeff Moyer957ac8c2017-11-14 20:37:27 -05001482 max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
Ross Zwisler282a8e02017-02-22 15:39:50 -08001483
Dave Jiangf4200392017-02-22 15:40:06 -08001484 trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
Ross Zwisler282a8e02017-02-22 15:39:50 -08001485
Ross Zwislerfffa2812017-08-25 15:55:36 -07001486 /*
1487 * Make sure that the faulting address's PMD offset (color) matches
1488 * the PMD offset from the start of the file. This is necessary so
1489 * that a PMD range in the page table overlaps exactly with a PMD
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001490 * range in the page cache.
Ross Zwislerfffa2812017-08-25 15:55:36 -07001491 */
1492 if ((vmf->pgoff & PG_PMD_COLOUR) !=
1493 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
1494 goto fallback;
1495
Ross Zwisler642261a2016-11-08 11:34:45 +11001496 /* Fall back to PTEs if we're going to COW */
1497 if (write && !(vma->vm_flags & VM_SHARED))
1498 goto fallback;
1499
1500 /* If the PMD would extend outside the VMA */
1501 if (pmd_addr < vma->vm_start)
1502 goto fallback;
1503 if ((pmd_addr + PMD_SIZE) > vma->vm_end)
1504 goto fallback;
1505
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001506 if (xas.xa_index >= max_pgoff) {
Ross Zwisler282a8e02017-02-22 15:39:50 -08001507 result = VM_FAULT_SIGBUS;
1508 goto out;
1509 }
Ross Zwisler642261a2016-11-08 11:34:45 +11001510
1511 /* If the PMD would extend beyond the file size */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001512 if ((xas.xa_index | PG_PMD_COLOUR) >= max_pgoff)
Ross Zwisler642261a2016-11-08 11:34:45 +11001513 goto fallback;
1514
1515 /*
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001516 * grab_mapping_entry() will make sure we get an empty PMD entry,
1517 * a zero PMD entry or a DAX PMD. If it can't (because a PTE
1518 * entry is already in the array, for instance), it will return
1519 * VM_FAULT_FALLBACK.
Jan Kara9f141d62016-10-19 14:34:31 +02001520 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001521 entry = grab_mapping_entry(&xas, mapping, DAX_PMD);
1522 if (xa_is_internal(entry)) {
1523 result = xa_to_internal(entry);
Ross Zwisler876f2942017-05-12 15:47:00 -07001524 goto fallback;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001525 }
Ross Zwisler876f2942017-05-12 15:47:00 -07001526
1527 /*
Ross Zwislere2093922017-06-02 14:46:37 -07001528 * It is possible, particularly with mixed reads & writes to private
1529 * mappings, that we have raced with a PTE fault that overlaps with
1530 * the PMD we need to set up. If so just return and the fault will be
1531 * retried.
1532 */
1533 if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
1534 !pmd_devmap(*vmf->pmd)) {
1535 result = 0;
1536 goto unlock_entry;
1537 }
1538
1539 /*
Ross Zwisler876f2942017-05-12 15:47:00 -07001540 * Note that we don't use iomap_apply here. We aren't doing I/O, only
1541 * setting up a mapping, so really we're using iomap_begin() as a way
1542 * to look up our filesystem block.
1543 */
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001544 pos = (loff_t)xas.xa_index << PAGE_SHIFT;
Ross Zwisler876f2942017-05-12 15:47:00 -07001545 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
1546 if (error)
1547 goto unlock_entry;
1548
1549 if (iomap.offset + iomap.length < pos + PMD_SIZE)
Jan Kara9f141d62016-10-19 14:34:31 +02001550 goto finish_iomap;
1551
Dan Williamsaaa422c2017-11-13 16:38:44 -08001552 sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
Jan Karacaa51d22017-11-01 16:36:42 +01001553
Ross Zwisler642261a2016-11-08 11:34:45 +11001554 switch (iomap.type) {
1555 case IOMAP_MAPPED:
Jan Kara302a5e32017-11-01 16:36:37 +01001556 error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
1557 if (error < 0)
1558 goto finish_iomap;
1559
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001560 entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
Matthew Wilcox3159f942017-11-03 13:30:42 -04001561 DAX_PMD, write && !sync);
Jan Kara302a5e32017-11-01 16:36:37 +01001562
Jan Karacaa51d22017-11-01 16:36:42 +01001563 /*
1564 * If we are doing synchronous page fault and inode needs fsync,
1565 * we can insert PMD into page tables only after that happens.
1566 * Skip insertion for now and return the pfn so that caller can
1567 * insert it after fsync is done.
1568 */
1569 if (sync) {
1570 if (WARN_ON_ONCE(!pfnp))
1571 goto finish_iomap;
1572 *pfnp = pfn;
1573 result = VM_FAULT_NEEDDSYNC;
1574 goto finish_iomap;
1575 }
1576
Jan Kara302a5e32017-11-01 16:36:37 +01001577 trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
Dan Williamsfce86ff2019-05-13 17:15:33 -07001578 result = vmf_insert_pfn_pmd(vmf, pfn, write);
Ross Zwisler642261a2016-11-08 11:34:45 +11001579 break;
1580 case IOMAP_UNWRITTEN:
1581 case IOMAP_HOLE:
1582 if (WARN_ON_ONCE(write))
Ross Zwisler876f2942017-05-12 15:47:00 -07001583 break;
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001584 result = dax_pmd_load_hole(&xas, vmf, &iomap, &entry);
Ross Zwisler642261a2016-11-08 11:34:45 +11001585 break;
1586 default:
1587 WARN_ON_ONCE(1);
1588 break;
1589 }
1590
Jan Kara9f141d62016-10-19 14:34:31 +02001591 finish_iomap:
1592 if (ops->iomap_end) {
1593 int copied = PMD_SIZE;
1594
1595 if (result == VM_FAULT_FALLBACK)
1596 copied = 0;
1597 /*
1598 * The fault is done by now and there's no way back (other
1599 * thread may be already happily using PMD we have installed).
1600 * Just ignore error from ->iomap_end since we cannot do much
1601 * with it.
1602 */
1603 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
1604 &iomap);
1605 }
Ross Zwisler876f2942017-05-12 15:47:00 -07001606 unlock_entry:
Matthew Wilcoxb15cd802018-03-29 22:58:27 -04001607 dax_unlock_entry(&xas, entry);
Ross Zwisler642261a2016-11-08 11:34:45 +11001608 fallback:
1609 if (result == VM_FAULT_FALLBACK) {
Dave Jiangd8a849e2017-02-22 15:40:03 -08001610 split_huge_pmd(vma, vmf->pmd, vmf->address);
Ross Zwisler642261a2016-11-08 11:34:45 +11001611 count_vm_event(THP_FAULT_FALLBACK);
1612 }
Ross Zwisler282a8e02017-02-22 15:39:50 -08001613out:
Dave Jiangf4200392017-02-22 15:40:06 -08001614 trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
Ross Zwisler642261a2016-11-08 11:34:45 +11001615 return result;
1616}
Dave Jianga2d58162017-02-24 14:56:59 -08001617#else
Souptick Joarderab77dab2018-06-07 17:04:29 -07001618static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
Arnd Bergmann01cddfe2017-02-27 14:26:44 -08001619 const struct iomap_ops *ops)
Dave Jianga2d58162017-02-24 14:56:59 -08001620{
1621 return VM_FAULT_FALLBACK;
1622}
Ross Zwisler642261a2016-11-08 11:34:45 +11001623#endif /* CONFIG_FS_DAX_PMD */
Dave Jianga2d58162017-02-24 14:56:59 -08001624
1625/**
1626 * dax_iomap_fault - handle a page fault on a DAX file
1627 * @vmf: The description of the fault
Jan Karacec04e82017-11-01 16:36:38 +01001628 * @pe_size: Size of the page to fault in
Jan Kara9a0dd422017-11-01 16:36:39 +01001629 * @pfnp: PFN to insert for synchronous faults if fsync is required
Jan Karac0b24622018-01-07 16:38:43 -05001630 * @iomap_errp: Storage for detailed error code in case of error
Jan Karacec04e82017-11-01 16:36:38 +01001631 * @ops: Iomap ops passed from the file system
Dave Jianga2d58162017-02-24 14:56:59 -08001632 *
1633 * When a page fault occurs, filesystems may call this helper in
1634 * their fault handler for DAX files. dax_iomap_fault() assumes the caller
1635 * has done all the necessary locking for page fault to proceed
1636 * successfully.
1637 */
Souptick Joarderab77dab2018-06-07 17:04:29 -07001638vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
Jan Karac0b24622018-01-07 16:38:43 -05001639 pfn_t *pfnp, int *iomap_errp, const struct iomap_ops *ops)
Dave Jianga2d58162017-02-24 14:56:59 -08001640{
Dave Jiangc791ace2017-02-24 14:57:08 -08001641 switch (pe_size) {
1642 case PE_SIZE_PTE:
Jan Karac0b24622018-01-07 16:38:43 -05001643 return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);
Dave Jiangc791ace2017-02-24 14:57:08 -08001644 case PE_SIZE_PMD:
Jan Kara9a0dd422017-11-01 16:36:39 +01001645 return dax_iomap_pmd_fault(vmf, pfnp, ops);
Dave Jianga2d58162017-02-24 14:56:59 -08001646 default:
1647 return VM_FAULT_FALLBACK;
1648 }
1649}
1650EXPORT_SYMBOL_GPL(dax_iomap_fault);
Jan Kara71eab6d2017-11-01 16:36:43 +01001651
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001652/*
Jan Kara71eab6d2017-11-01 16:36:43 +01001653 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
1654 * @vmf: The description of the fault
Jan Kara71eab6d2017-11-01 16:36:43 +01001655 * @pfn: PFN to insert
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001656 * @order: Order of entry to insert.
Jan Kara71eab6d2017-11-01 16:36:43 +01001657 *
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001658 * This function inserts a writeable PTE or PMD entry into the page tables
1659 * for an mmaped DAX file. It also marks the page cache entry as dirty.
Jan Kara71eab6d2017-11-01 16:36:43 +01001660 */
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001661static vm_fault_t
1662dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)
Jan Kara71eab6d2017-11-01 16:36:43 +01001663{
1664 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001665 XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);
1666 void *entry;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001667 vm_fault_t ret;
Jan Kara71eab6d2017-11-01 16:36:43 +01001668
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001669 xas_lock_irq(&xas);
1670 entry = get_unlocked_entry(&xas);
Jan Kara71eab6d2017-11-01 16:36:43 +01001671 /* Did we race with someone splitting entry or so? */
1672 if (!entry ||
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001673 (order == 0 && !dax_is_pte_entry(entry)) ||
Matthew Wilcox0e40de02018-11-16 15:19:13 -05001674 (order == PMD_ORDER && !dax_is_pmd_entry(entry))) {
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001675 put_unlocked_entry(&xas, entry);
1676 xas_unlock_irq(&xas);
Jan Kara71eab6d2017-11-01 16:36:43 +01001677 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
1678 VM_FAULT_NOPAGE);
1679 return VM_FAULT_NOPAGE;
1680 }
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001681 xas_set_mark(&xas, PAGECACHE_TAG_DIRTY);
1682 dax_lock_entry(&xas, entry);
1683 xas_unlock_irq(&xas);
1684 if (order == 0)
Souptick Joarderab77dab2018-06-07 17:04:29 -07001685 ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
Jan Kara71eab6d2017-11-01 16:36:43 +01001686#ifdef CONFIG_FS_DAX_PMD
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001687 else if (order == PMD_ORDER)
Dan Williamsfce86ff2019-05-13 17:15:33 -07001688 ret = vmf_insert_pfn_pmd(vmf, pfn, FAULT_FLAG_WRITE);
Jan Kara71eab6d2017-11-01 16:36:43 +01001689#endif
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001690 else
Souptick Joarderab77dab2018-06-07 17:04:29 -07001691 ret = VM_FAULT_FALLBACK;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001692 dax_unlock_entry(&xas, entry);
Souptick Joarderab77dab2018-06-07 17:04:29 -07001693 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
1694 return ret;
Jan Kara71eab6d2017-11-01 16:36:43 +01001695}
1696
1697/**
1698 * dax_finish_sync_fault - finish synchronous page fault
1699 * @vmf: The description of the fault
1700 * @pe_size: Size of entry to be inserted
1701 * @pfn: PFN to insert
1702 *
1703 * This function ensures that the file range touched by the page fault is
1704 * stored persistently on the media and handles inserting of appropriate page
1705 * table entry.
1706 */
Souptick Joarderab77dab2018-06-07 17:04:29 -07001707vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
1708 enum page_entry_size pe_size, pfn_t pfn)
Jan Kara71eab6d2017-11-01 16:36:43 +01001709{
1710 int err;
1711 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001712 unsigned int order = pe_order(pe_size);
1713 size_t len = PAGE_SIZE << order;
Jan Kara71eab6d2017-11-01 16:36:43 +01001714
Jan Kara71eab6d2017-11-01 16:36:43 +01001715 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
1716 if (err)
1717 return VM_FAULT_SIGBUS;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001718 return dax_insert_pfn_mkwrite(vmf, pfn, order);
Jan Kara71eab6d2017-11-01 16:36:43 +01001719}
1720EXPORT_SYMBOL_GPL(dax_finish_sync_fault);