blob: 4ed58c9a94a9916959e304d28f77e9002b64e8df [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
David Sterbac1d7c512018-04-03 19:23:33 +02002
Chris Masond1310b22008-01-24 16:13:08 -05003#include <linux/bitops.h>
4#include <linux/slab.h>
5#include <linux/bio.h>
6#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/pagemap.h>
8#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05009#include <linux/spinlock.h>
10#include <linux/blkdev.h>
11#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050012#include <linux/writeback.h>
13#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070014#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060015#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050016#include "extent_io.h"
17#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040018#include "ctree.h"
19#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020020#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010021#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040022#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040023#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080024#include "backref.h"
David Sterba6af49db2017-06-23 04:09:57 +020025#include "disk-io.h"
Chris Masond1310b22008-01-24 16:13:08 -050026
Chris Masond1310b22008-01-24 16:13:08 -050027static struct kmem_cache *extent_state_cache;
28static struct kmem_cache *extent_buffer_cache;
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -040029static struct bio_set btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050030
Filipe Manana27a35072014-07-06 20:09:59 +010031static inline bool extent_state_in_tree(const struct extent_state *state)
32{
33 return !RB_EMPTY_NODE(&state->rb_node);
34}
35
Eric Sandeen6d49ba12013-04-22 16:12:31 +000036#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050037static LIST_HEAD(buffers);
38static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040039
Chris Masond3977122009-01-05 21:25:51 -050040static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000041
42static inline
43void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
44{
45 unsigned long flags;
46
47 spin_lock_irqsave(&leak_lock, flags);
48 list_add(new, head);
49 spin_unlock_irqrestore(&leak_lock, flags);
50}
51
52static inline
53void btrfs_leak_debug_del(struct list_head *entry)
54{
55 unsigned long flags;
56
57 spin_lock_irqsave(&leak_lock, flags);
58 list_del(entry);
59 spin_unlock_irqrestore(&leak_lock, flags);
60}
61
62static inline
63void btrfs_leak_debug_check(void)
64{
65 struct extent_state *state;
66 struct extent_buffer *eb;
67
68 while (!list_empty(&states)) {
69 state = list_entry(states.next, struct extent_state, leak_list);
David Sterba9ee49a042015-01-14 19:52:13 +010070 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
Filipe Manana27a35072014-07-06 20:09:59 +010071 state->start, state->end, state->state,
72 extent_state_in_tree(state),
Elena Reshetovab7ac31b2017-03-03 10:55:19 +020073 refcount_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000074 list_del(&state->leak_list);
75 kmem_cache_free(extent_state_cache, state);
76 }
77
78 while (!list_empty(&buffers)) {
79 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Liu Boaf2679e2018-01-25 11:02:48 -070080 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
81 eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000082 list_del(&eb->leak_list);
83 kmem_cache_free(extent_buffer_cache, eb);
84 }
85}
David Sterba8d599ae2013-04-30 15:22:23 +000086
Josef Bacika5dee372013-12-13 10:02:44 -050087#define btrfs_debug_check_extent_io_range(tree, start, end) \
88 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000089static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050090 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000091{
Nikolay Borisov65a680f2018-11-01 14:09:49 +020092 struct inode *inode = tree->private_data;
93 u64 isize;
94
95 if (!inode || !is_data_inode(inode))
96 return;
97
98 isize = i_size_read(inode);
99 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
100 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
101 "%s: ino %llu isize %llu odd range [%llu,%llu]",
102 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
103 }
David Sterba8d599ae2013-04-30 15:22:23 +0000104}
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000105#else
106#define btrfs_leak_debug_add(new, head) do {} while (0)
107#define btrfs_leak_debug_del(entry) do {} while (0)
108#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000109#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400110#endif
Chris Masond1310b22008-01-24 16:13:08 -0500111
Chris Masond1310b22008-01-24 16:13:08 -0500112#define BUFFER_LRU_MAX 64
113
114struct tree_entry {
115 u64 start;
116 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500117 struct rb_node rb_node;
118};
119
120struct extent_page_data {
121 struct bio *bio;
122 struct extent_io_tree *tree;
Chris Mason771ed682008-11-06 22:02:51 -0500123 /* tells writepage not to lock the state bits for this range
124 * it still does the unlocking
125 */
Chris Masonffbd5172009-04-20 15:50:09 -0400126 unsigned int extent_locked:1;
127
Christoph Hellwig70fd7612016-11-01 07:40:10 -0600128 /* tells the submit_bio code to use REQ_SYNC */
Chris Masonffbd5172009-04-20 15:50:09 -0400129 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500130};
131
David Sterba57599c72018-03-01 17:56:34 +0100132static int add_extent_changeset(struct extent_state *state, unsigned bits,
Qu Wenruod38ed272015-10-12 14:53:37 +0800133 struct extent_changeset *changeset,
134 int set)
135{
136 int ret;
137
138 if (!changeset)
David Sterba57599c72018-03-01 17:56:34 +0100139 return 0;
Qu Wenruod38ed272015-10-12 14:53:37 +0800140 if (set && (state->state & bits) == bits)
David Sterba57599c72018-03-01 17:56:34 +0100141 return 0;
Qu Wenruofefdc552015-10-12 15:35:38 +0800142 if (!set && (state->state & bits) == 0)
David Sterba57599c72018-03-01 17:56:34 +0100143 return 0;
Qu Wenruod38ed272015-10-12 14:53:37 +0800144 changeset->bytes_changed += state->end - state->start + 1;
David Sterba53d32352017-02-13 13:42:29 +0100145 ret = ulist_add(&changeset->range_changed, state->start, state->end,
Qu Wenruod38ed272015-10-12 14:53:37 +0800146 GFP_ATOMIC);
David Sterba57599c72018-03-01 17:56:34 +0100147 return ret;
Qu Wenruod38ed272015-10-12 14:53:37 +0800148}
149
David Sterbaaab6e9e2017-11-30 18:00:02 +0100150static void flush_write_bio(struct extent_page_data *epd);
David Sterbae2932ee2017-06-23 04:16:17 +0200151
Chris Masond1310b22008-01-24 16:13:08 -0500152int __init extent_io_init(void)
153{
David Sterba837e1972012-09-07 03:00:48 -0600154 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200155 sizeof(struct extent_state), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300156 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500157 if (!extent_state_cache)
158 return -ENOMEM;
159
David Sterba837e1972012-09-07 03:00:48 -0600160 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200161 sizeof(struct extent_buffer), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300162 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500163 if (!extent_buffer_cache)
164 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400165
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400166 if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
167 offsetof(struct btrfs_io_bio, bio),
168 BIOSET_NEED_BVECS))
Chris Mason9be33952013-05-17 18:30:14 -0400169 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700170
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400171 if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700172 goto free_bioset;
173
Chris Masond1310b22008-01-24 16:13:08 -0500174 return 0;
175
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700176free_bioset:
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400177 bioset_exit(&btrfs_bioset);
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700178
Chris Mason9be33952013-05-17 18:30:14 -0400179free_buffer_cache:
180 kmem_cache_destroy(extent_buffer_cache);
181 extent_buffer_cache = NULL;
182
Chris Masond1310b22008-01-24 16:13:08 -0500183free_state_cache:
184 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400185 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500186 return -ENOMEM;
187}
188
David Sterbae67c7182018-02-19 17:24:18 +0100189void __cold extent_io_exit(void)
Chris Masond1310b22008-01-24 16:13:08 -0500190{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000191 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000192
193 /*
194 * Make sure all delayed rcu free are flushed before we
195 * destroy caches.
196 */
197 rcu_barrier();
Kinglong Mee5598e902016-01-29 21:36:35 +0800198 kmem_cache_destroy(extent_state_cache);
199 kmem_cache_destroy(extent_buffer_cache);
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400200 bioset_exit(&btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500201}
202
203void extent_io_tree_init(struct extent_io_tree *tree,
Josef Bacikc6100a42017-05-05 11:57:13 -0400204 void *private_data)
Chris Masond1310b22008-01-24 16:13:08 -0500205{
Eric Paris6bef4d32010-02-23 19:43:04 +0000206 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500207 tree->ops = NULL;
208 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500209 spin_lock_init(&tree->lock);
Josef Bacikc6100a42017-05-05 11:57:13 -0400210 tree->private_data = private_data;
Chris Masond1310b22008-01-24 16:13:08 -0500211}
Chris Masond1310b22008-01-24 16:13:08 -0500212
Christoph Hellwigb2950862008-12-02 09:54:17 -0500213static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500214{
215 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500216
Michal Hocko3ba7ab22017-01-09 15:39:02 +0100217 /*
218 * The given mask might be not appropriate for the slab allocator,
219 * drop the unsupported bits
220 */
221 mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
Chris Masond1310b22008-01-24 16:13:08 -0500222 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400223 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500224 return state;
225 state->state = 0;
David Sterba47dc1962016-02-11 13:24:13 +0100226 state->failrec = NULL;
Filipe Manana27a35072014-07-06 20:09:59 +0100227 RB_CLEAR_NODE(&state->rb_node);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000228 btrfs_leak_debug_add(&state->leak_list, &states);
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200229 refcount_set(&state->refs, 1);
Chris Masond1310b22008-01-24 16:13:08 -0500230 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100231 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500232 return state;
233}
Chris Masond1310b22008-01-24 16:13:08 -0500234
Chris Mason4845e442010-05-25 20:56:50 -0400235void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500236{
Chris Masond1310b22008-01-24 16:13:08 -0500237 if (!state)
238 return;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200239 if (refcount_dec_and_test(&state->refs)) {
Filipe Manana27a35072014-07-06 20:09:59 +0100240 WARN_ON(extent_state_in_tree(state));
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000241 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100242 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500243 kmem_cache_free(extent_state_cache, state);
244 }
245}
Chris Masond1310b22008-01-24 16:13:08 -0500246
Filipe Mananaf2071b22014-02-12 15:05:53 +0000247static struct rb_node *tree_insert(struct rb_root *root,
248 struct rb_node *search_start,
249 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000250 struct rb_node *node,
251 struct rb_node ***p_in,
252 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500253{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000254 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500255 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500256 struct tree_entry *entry;
257
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000258 if (p_in && parent_in) {
259 p = *p_in;
260 parent = *parent_in;
261 goto do_insert;
262 }
263
Filipe Mananaf2071b22014-02-12 15:05:53 +0000264 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500265 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500266 parent = *p;
267 entry = rb_entry(parent, struct tree_entry, rb_node);
268
269 if (offset < entry->start)
270 p = &(*p)->rb_left;
271 else if (offset > entry->end)
272 p = &(*p)->rb_right;
273 else
274 return parent;
275 }
276
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000277do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500278 rb_link_node(node, parent, p);
279 rb_insert_color(node, root);
280 return NULL;
281}
282
Chris Mason80ea96b2008-02-01 14:51:59 -0500283static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000284 struct rb_node **prev_ret,
285 struct rb_node **next_ret,
286 struct rb_node ***p_ret,
287 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500288{
Chris Mason80ea96b2008-02-01 14:51:59 -0500289 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000290 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500291 struct rb_node *prev = NULL;
292 struct rb_node *orig_prev = NULL;
293 struct tree_entry *entry;
294 struct tree_entry *prev_entry = NULL;
295
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000296 while (*n) {
297 prev = *n;
298 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500299 prev_entry = entry;
300
301 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000302 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500303 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000304 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500305 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000306 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500307 }
308
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000309 if (p_ret)
310 *p_ret = n;
311 if (parent_ret)
312 *parent_ret = prev;
313
Chris Masond1310b22008-01-24 16:13:08 -0500314 if (prev_ret) {
315 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500316 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500317 prev = rb_next(prev);
318 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
319 }
320 *prev_ret = prev;
321 prev = orig_prev;
322 }
323
324 if (next_ret) {
325 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500326 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500327 prev = rb_prev(prev);
328 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
329 }
330 *next_ret = prev;
331 }
332 return NULL;
333}
334
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000335static inline struct rb_node *
336tree_search_for_insert(struct extent_io_tree *tree,
337 u64 offset,
338 struct rb_node ***p_ret,
339 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500340{
Chris Mason70dec802008-01-29 09:59:12 -0500341 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500342 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500343
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000344 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500345 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500346 return prev;
347 return ret;
348}
349
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000350static inline struct rb_node *tree_search(struct extent_io_tree *tree,
351 u64 offset)
352{
353 return tree_search_for_insert(tree, offset, NULL, NULL);
354}
355
Chris Masond1310b22008-01-24 16:13:08 -0500356/*
357 * utility function to look for merge candidates inside a given range.
358 * Any extents with matching state are merged together into a single
359 * extent in the tree. Extents with EXTENT_IO in their state field
360 * are not merged because the end_io handlers need to be able to do
361 * operations on them without sleeping (or doing allocations/splits).
362 *
363 * This should be called with the tree lock held.
364 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000365static void merge_state(struct extent_io_tree *tree,
366 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500367{
368 struct extent_state *other;
369 struct rb_node *other_node;
370
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400371 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000372 return;
Chris Masond1310b22008-01-24 16:13:08 -0500373
374 other_node = rb_prev(&state->rb_node);
375 if (other_node) {
376 other = rb_entry(other_node, struct extent_state, rb_node);
377 if (other->end == state->start - 1 &&
378 other->state == state->state) {
Nikolay Borisov5c848192018-11-01 14:09:52 +0200379 if (tree->private_data &&
380 is_data_inode(tree->private_data))
381 btrfs_merge_delalloc_extent(tree->private_data,
382 state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500383 state->start = other->start;
Chris Masond1310b22008-01-24 16:13:08 -0500384 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100385 RB_CLEAR_NODE(&other->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500386 free_extent_state(other);
387 }
388 }
389 other_node = rb_next(&state->rb_node);
390 if (other_node) {
391 other = rb_entry(other_node, struct extent_state, rb_node);
392 if (other->start == state->end + 1 &&
393 other->state == state->state) {
Nikolay Borisov5c848192018-11-01 14:09:52 +0200394 if (tree->private_data &&
395 is_data_inode(tree->private_data))
396 btrfs_merge_delalloc_extent(tree->private_data,
397 state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400398 state->end = other->end;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400399 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100400 RB_CLEAR_NODE(&other->rb_node);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400401 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500402 }
403 }
Chris Masond1310b22008-01-24 16:13:08 -0500404}
405
Xiao Guangrong3150b692011-07-14 03:19:08 +0000406static void set_state_bits(struct extent_io_tree *tree,
Qu Wenruod38ed272015-10-12 14:53:37 +0800407 struct extent_state *state, unsigned *bits,
408 struct extent_changeset *changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000409
Chris Masond1310b22008-01-24 16:13:08 -0500410/*
411 * insert an extent_state struct into the tree. 'bits' are set on the
412 * struct before it is inserted.
413 *
414 * This may return -EEXIST if the extent is already there, in which case the
415 * state struct is freed.
416 *
417 * The tree lock is not taken internally. This is a utility function and
418 * probably isn't what you want to call (see set/clear_extent_bit).
419 */
420static int insert_state(struct extent_io_tree *tree,
421 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000422 struct rb_node ***p,
423 struct rb_node **parent,
Qu Wenruod38ed272015-10-12 14:53:37 +0800424 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500425{
426 struct rb_node *node;
427
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000428 if (end < start)
Frank Holtonefe120a2013-12-20 11:37:06 -0500429 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200430 end, start);
Chris Masond1310b22008-01-24 16:13:08 -0500431 state->start = start;
432 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400433
Qu Wenruod38ed272015-10-12 14:53:37 +0800434 set_state_bits(tree, state, bits, changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000435
Filipe Mananaf2071b22014-02-12 15:05:53 +0000436 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500437 if (node) {
438 struct extent_state *found;
439 found = rb_entry(node, struct extent_state, rb_node);
Jeff Mahoney62e85572016-09-20 10:05:01 -0400440 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200441 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500442 return -EEXIST;
443 }
444 merge_state(tree, state);
445 return 0;
446}
447
448/*
449 * split a given extent state struct in two, inserting the preallocated
450 * struct 'prealloc' as the newly created second half. 'split' indicates an
451 * offset inside 'orig' where it should be split.
452 *
453 * Before calling,
454 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
455 * are two extent state structs in the tree:
456 * prealloc: [orig->start, split - 1]
457 * orig: [ split, orig->end ]
458 *
459 * The tree locks are not taken by this function. They need to be held
460 * by the caller.
461 */
462static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
463 struct extent_state *prealloc, u64 split)
464{
465 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400466
Nikolay Borisovabbb55f2018-11-01 14:09:53 +0200467 if (tree->private_data && is_data_inode(tree->private_data))
468 btrfs_split_delalloc_extent(tree->private_data, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400469
Chris Masond1310b22008-01-24 16:13:08 -0500470 prealloc->start = orig->start;
471 prealloc->end = split - 1;
472 prealloc->state = orig->state;
473 orig->start = split;
474
Filipe Mananaf2071b22014-02-12 15:05:53 +0000475 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
476 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500477 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500478 free_extent_state(prealloc);
479 return -EEXIST;
480 }
481 return 0;
482}
483
Li Zefancdc6a392012-03-12 16:39:48 +0800484static struct extent_state *next_state(struct extent_state *state)
485{
486 struct rb_node *next = rb_next(&state->rb_node);
487 if (next)
488 return rb_entry(next, struct extent_state, rb_node);
489 else
490 return NULL;
491}
492
Chris Masond1310b22008-01-24 16:13:08 -0500493/*
494 * utility function to clear some bits in an extent state struct.
Andrea Gelmini52042d82018-11-28 12:05:13 +0100495 * it will optionally wake up anyone waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500496 *
497 * If no bits are set on the state struct after clearing things, the
498 * struct is freed and removed from the tree
499 */
Li Zefancdc6a392012-03-12 16:39:48 +0800500static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
501 struct extent_state *state,
Qu Wenruofefdc552015-10-12 15:35:38 +0800502 unsigned *bits, int wake,
503 struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500504{
Li Zefancdc6a392012-03-12 16:39:48 +0800505 struct extent_state *next;
David Sterba9ee49a042015-01-14 19:52:13 +0100506 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
David Sterba57599c72018-03-01 17:56:34 +0100507 int ret;
Chris Masond1310b22008-01-24 16:13:08 -0500508
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400509 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500510 u64 range = state->end - state->start + 1;
511 WARN_ON(range > tree->dirty_bytes);
512 tree->dirty_bytes -= range;
513 }
Nikolay Borisova36bb5f2018-11-01 14:09:51 +0200514
515 if (tree->private_data && is_data_inode(tree->private_data))
516 btrfs_clear_delalloc_extent(tree->private_data, state, bits);
517
David Sterba57599c72018-03-01 17:56:34 +0100518 ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
519 BUG_ON(ret < 0);
Josef Bacik32c00af2009-10-08 13:34:05 -0400520 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500521 if (wake)
522 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400523 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800524 next = next_state(state);
Filipe Manana27a35072014-07-06 20:09:59 +0100525 if (extent_state_in_tree(state)) {
Chris Masond1310b22008-01-24 16:13:08 -0500526 rb_erase(&state->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100527 RB_CLEAR_NODE(&state->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500528 free_extent_state(state);
529 } else {
530 WARN_ON(1);
531 }
532 } else {
533 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800534 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500535 }
Li Zefancdc6a392012-03-12 16:39:48 +0800536 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500537}
538
Xiao Guangrong82337672011-04-20 06:44:57 +0000539static struct extent_state *
540alloc_extent_state_atomic(struct extent_state *prealloc)
541{
542 if (!prealloc)
543 prealloc = alloc_extent_state(GFP_ATOMIC);
544
545 return prealloc;
546}
547
Eric Sandeen48a3b632013-04-25 20:41:01 +0000548static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400549{
David Sterba05912a32018-07-18 19:23:45 +0200550 struct inode *inode = tree->private_data;
551
552 btrfs_panic(btrfs_sb(inode->i_sb), err,
553 "locking error: extent tree was modified by another thread while locked");
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400554}
555
Chris Masond1310b22008-01-24 16:13:08 -0500556/*
557 * clear some bits on a range in the tree. This may require splitting
558 * or inserting elements in the tree, so the gfp mask is used to
559 * indicate which allocations or sleeping are allowed.
560 *
561 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
562 * the given range from the tree regardless of state (ie for truncate).
563 *
564 * the range [start, end] is inclusive.
565 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100566 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500567 */
David Sterba66b0c882017-10-31 16:30:47 +0100568int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
Qu Wenruofefdc552015-10-12 15:35:38 +0800569 unsigned bits, int wake, int delete,
570 struct extent_state **cached_state,
571 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500572{
573 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400574 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500575 struct extent_state *prealloc = NULL;
576 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400577 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500578 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000579 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500580
Josef Bacika5dee372013-12-13 10:02:44 -0500581 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000582
Josef Bacik7ee9e442013-06-21 16:37:03 -0400583 if (bits & EXTENT_DELALLOC)
584 bits |= EXTENT_NORESERVE;
585
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400586 if (delete)
587 bits |= ~EXTENT_CTLBITS;
588 bits |= EXTENT_FIRST_DELALLOC;
589
Josef Bacik2ac55d42010-02-03 19:33:23 +0000590 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
591 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500592again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800593 if (!prealloc && gfpflags_allow_blocking(mask)) {
Filipe Mananac7bc6312014-11-03 14:12:57 +0000594 /*
595 * Don't care for allocation failure here because we might end
596 * up not needing the pre-allocated extent state at all, which
597 * is the case if we only have in the tree extent states that
598 * cover our input range and don't cover too any other range.
599 * If we end up needing a new extent state we allocate it later.
600 */
Chris Masond1310b22008-01-24 16:13:08 -0500601 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500602 }
603
Chris Masoncad321a2008-12-17 14:51:42 -0500604 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400605 if (cached_state) {
606 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000607
608 if (clear) {
609 *cached_state = NULL;
610 cached_state = NULL;
611 }
612
Filipe Manana27a35072014-07-06 20:09:59 +0100613 if (cached && extent_state_in_tree(cached) &&
614 cached->start <= start && cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000615 if (clear)
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200616 refcount_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400617 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400618 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400619 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000620 if (clear)
621 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400622 }
Chris Masond1310b22008-01-24 16:13:08 -0500623 /*
624 * this search will find the extents that end after
625 * our range starts
626 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500627 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500628 if (!node)
629 goto out;
630 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400631hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500632 if (state->start > end)
633 goto out;
634 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400635 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500636
Liu Bo04493142012-02-16 18:34:37 +0800637 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800638 if (!(state->state & bits)) {
639 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800640 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800641 }
Liu Bo04493142012-02-16 18:34:37 +0800642
Chris Masond1310b22008-01-24 16:13:08 -0500643 /*
644 * | ---- desired range ---- |
645 * | state | or
646 * | ------------- state -------------- |
647 *
648 * We need to split the extent we found, and may flip
649 * bits on second half.
650 *
651 * If the extent we found extends past our range, we
652 * just split and search again. It'll get split again
653 * the next time though.
654 *
655 * If the extent we found is inside our range, we clear
656 * the desired bit on it.
657 */
658
659 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000660 prealloc = alloc_extent_state_atomic(prealloc);
661 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500662 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400663 if (err)
664 extent_io_tree_panic(tree, err);
665
Chris Masond1310b22008-01-24 16:13:08 -0500666 prealloc = NULL;
667 if (err)
668 goto out;
669 if (state->end <= end) {
Qu Wenruofefdc552015-10-12 15:35:38 +0800670 state = clear_state_bit(tree, state, &bits, wake,
671 changeset);
Liu Bod1ac6e42012-05-10 18:10:39 +0800672 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500673 }
674 goto search_again;
675 }
676 /*
677 * | ---- desired range ---- |
678 * | state |
679 * We need to split the extent, and clear the bit
680 * on the first half
681 */
682 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000683 prealloc = alloc_extent_state_atomic(prealloc);
684 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500685 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400686 if (err)
687 extent_io_tree_panic(tree, err);
688
Chris Masond1310b22008-01-24 16:13:08 -0500689 if (wake)
690 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400691
Qu Wenruofefdc552015-10-12 15:35:38 +0800692 clear_state_bit(tree, prealloc, &bits, wake, changeset);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400693
Chris Masond1310b22008-01-24 16:13:08 -0500694 prealloc = NULL;
695 goto out;
696 }
Chris Mason42daec22009-09-23 19:51:09 -0400697
Qu Wenruofefdc552015-10-12 15:35:38 +0800698 state = clear_state_bit(tree, state, &bits, wake, changeset);
Liu Bo04493142012-02-16 18:34:37 +0800699next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400700 if (last_end == (u64)-1)
701 goto out;
702 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800703 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800704 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500705
706search_again:
707 if (start > end)
708 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500709 spin_unlock(&tree->lock);
Mel Gormand0164ad2015-11-06 16:28:21 -0800710 if (gfpflags_allow_blocking(mask))
Chris Masond1310b22008-01-24 16:13:08 -0500711 cond_resched();
712 goto again;
David Sterba7ab5cb22016-04-27 01:02:15 +0200713
714out:
715 spin_unlock(&tree->lock);
716 if (prealloc)
717 free_extent_state(prealloc);
718
719 return 0;
720
Chris Masond1310b22008-01-24 16:13:08 -0500721}
Chris Masond1310b22008-01-24 16:13:08 -0500722
Jeff Mahoney143bede2012-03-01 14:56:26 +0100723static void wait_on_state(struct extent_io_tree *tree,
724 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500725 __releases(tree->lock)
726 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500727{
728 DEFINE_WAIT(wait);
729 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500730 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500731 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500732 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500733 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500734}
735
736/*
737 * waits for one or more bits to clear on a range in the state tree.
738 * The range [start, end] is inclusive.
739 * The tree lock is taken by this function
740 */
David Sterba41074882013-04-29 13:38:46 +0000741static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
742 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500743{
744 struct extent_state *state;
745 struct rb_node *node;
746
Josef Bacika5dee372013-12-13 10:02:44 -0500747 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000748
Chris Masoncad321a2008-12-17 14:51:42 -0500749 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500750again:
751 while (1) {
752 /*
753 * this search will find all the extents that end after
754 * our range starts
755 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500756 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100757process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500758 if (!node)
759 break;
760
761 state = rb_entry(node, struct extent_state, rb_node);
762
763 if (state->start > end)
764 goto out;
765
766 if (state->state & bits) {
767 start = state->start;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200768 refcount_inc(&state->refs);
Chris Masond1310b22008-01-24 16:13:08 -0500769 wait_on_state(tree, state);
770 free_extent_state(state);
771 goto again;
772 }
773 start = state->end + 1;
774
775 if (start > end)
776 break;
777
Filipe Mananac50d3e72014-03-31 14:53:25 +0100778 if (!cond_resched_lock(&tree->lock)) {
779 node = rb_next(node);
780 goto process_node;
781 }
Chris Masond1310b22008-01-24 16:13:08 -0500782 }
783out:
Chris Masoncad321a2008-12-17 14:51:42 -0500784 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500785}
Chris Masond1310b22008-01-24 16:13:08 -0500786
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000787static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500788 struct extent_state *state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800789 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500790{
David Sterba9ee49a042015-01-14 19:52:13 +0100791 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
David Sterba57599c72018-03-01 17:56:34 +0100792 int ret;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400793
Nikolay Borisove06a1fc2018-11-01 14:09:50 +0200794 if (tree->private_data && is_data_inode(tree->private_data))
795 btrfs_set_delalloc_extent(tree->private_data, state, bits);
796
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400797 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500798 u64 range = state->end - state->start + 1;
799 tree->dirty_bytes += range;
800 }
David Sterba57599c72018-03-01 17:56:34 +0100801 ret = add_extent_changeset(state, bits_to_set, changeset, 1);
802 BUG_ON(ret < 0);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400803 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500804}
805
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100806static void cache_state_if_flags(struct extent_state *state,
807 struct extent_state **cached_ptr,
David Sterba9ee49a042015-01-14 19:52:13 +0100808 unsigned flags)
Chris Mason2c64c532009-09-02 15:04:12 -0400809{
810 if (cached_ptr && !(*cached_ptr)) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100811 if (!flags || (state->state & flags)) {
Chris Mason2c64c532009-09-02 15:04:12 -0400812 *cached_ptr = state;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200813 refcount_inc(&state->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400814 }
815 }
816}
817
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100818static void cache_state(struct extent_state *state,
819 struct extent_state **cached_ptr)
820{
821 return cache_state_if_flags(state, cached_ptr,
822 EXTENT_IOBITS | EXTENT_BOUNDARY);
823}
824
Chris Masond1310b22008-01-24 16:13:08 -0500825/*
Chris Mason1edbb732009-09-02 13:24:36 -0400826 * set some bits on a range in the tree. This may require allocations or
827 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500828 *
Chris Mason1edbb732009-09-02 13:24:36 -0400829 * If any of the exclusive bits are set, this will fail with -EEXIST if some
830 * part of the range already has the desired bits set. The start of the
831 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500832 *
Chris Mason1edbb732009-09-02 13:24:36 -0400833 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500834 */
Chris Mason1edbb732009-09-02 13:24:36 -0400835
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100836static int __must_check
837__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100838 unsigned bits, unsigned exclusive_bits,
David Sterba41074882013-04-29 13:38:46 +0000839 u64 *failed_start, struct extent_state **cached_state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800840 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500841{
842 struct extent_state *state;
843 struct extent_state *prealloc = NULL;
844 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000845 struct rb_node **p;
846 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500847 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500848 u64 last_start;
849 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400850
Josef Bacika5dee372013-12-13 10:02:44 -0500851 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000852
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400853 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500854again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800855 if (!prealloc && gfpflags_allow_blocking(mask)) {
David Sterba059f7912016-04-27 01:03:45 +0200856 /*
857 * Don't care for allocation failure here because we might end
858 * up not needing the pre-allocated extent state at all, which
859 * is the case if we only have in the tree extent states that
860 * cover our input range and don't cover too any other range.
861 * If we end up needing a new extent state we allocate it later.
862 */
Chris Masond1310b22008-01-24 16:13:08 -0500863 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500864 }
865
Chris Masoncad321a2008-12-17 14:51:42 -0500866 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400867 if (cached_state && *cached_state) {
868 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400869 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +0100870 extent_state_in_tree(state)) {
Chris Mason9655d292009-09-02 15:22:30 -0400871 node = &state->rb_node;
872 goto hit_next;
873 }
874 }
Chris Masond1310b22008-01-24 16:13:08 -0500875 /*
876 * this search will find all the extents that end after
877 * our range starts.
878 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000879 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -0500880 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000881 prealloc = alloc_extent_state_atomic(prealloc);
882 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000883 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +0800884 &p, &parent, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400885 if (err)
886 extent_io_tree_panic(tree, err);
887
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +0000888 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500889 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500890 goto out;
891 }
Chris Masond1310b22008-01-24 16:13:08 -0500892 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400893hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500894 last_start = state->start;
895 last_end = state->end;
896
897 /*
898 * | ---- desired range ---- |
899 * | state |
900 *
901 * Just lock what we found and keep going
902 */
903 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400904 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500905 *failed_start = state->start;
906 err = -EEXIST;
907 goto out;
908 }
Chris Mason42daec22009-09-23 19:51:09 -0400909
Qu Wenruod38ed272015-10-12 14:53:37 +0800910 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -0400911 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500912 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400913 if (last_end == (u64)-1)
914 goto out;
915 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800916 state = next_state(state);
917 if (start < end && state && state->start == start &&
918 !need_resched())
919 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500920 goto search_again;
921 }
922
923 /*
924 * | ---- desired range ---- |
925 * | state |
926 * or
927 * | ------------- state -------------- |
928 *
929 * We need to split the extent we found, and may flip bits on
930 * second half.
931 *
932 * If the extent we found extends past our
933 * range, we just split and search again. It'll get split
934 * again the next time though.
935 *
936 * If the extent we found is inside our range, we set the
937 * desired bit on it.
938 */
939 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400940 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500941 *failed_start = start;
942 err = -EEXIST;
943 goto out;
944 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000945
946 prealloc = alloc_extent_state_atomic(prealloc);
947 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500948 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400949 if (err)
950 extent_io_tree_panic(tree, err);
951
Chris Masond1310b22008-01-24 16:13:08 -0500952 prealloc = NULL;
953 if (err)
954 goto out;
955 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +0800956 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -0400957 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500958 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400959 if (last_end == (u64)-1)
960 goto out;
961 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800962 state = next_state(state);
963 if (start < end && state && state->start == start &&
964 !need_resched())
965 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500966 }
967 goto search_again;
968 }
969 /*
970 * | ---- desired range ---- |
971 * | state | or | state |
972 *
973 * There's a hole, we need to insert something in it and
974 * ignore the extent we found.
975 */
976 if (state->start > start) {
977 u64 this_end;
978 if (end < last_start)
979 this_end = end;
980 else
Chris Masond3977122009-01-05 21:25:51 -0500981 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000982
983 prealloc = alloc_extent_state_atomic(prealloc);
984 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000985
986 /*
987 * Avoid to free 'prealloc' if it can be merged with
988 * the later extent.
989 */
Chris Masond1310b22008-01-24 16:13:08 -0500990 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +0800991 NULL, NULL, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400992 if (err)
993 extent_io_tree_panic(tree, err);
994
Chris Mason2c64c532009-09-02 15:04:12 -0400995 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500996 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500997 start = this_end + 1;
998 goto search_again;
999 }
1000 /*
1001 * | ---- desired range ---- |
1002 * | state |
1003 * We need to split the extent, and set the bit
1004 * on the first half
1005 */
1006 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -04001007 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001008 *failed_start = start;
1009 err = -EEXIST;
1010 goto out;
1011 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001012
1013 prealloc = alloc_extent_state_atomic(prealloc);
1014 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001015 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001016 if (err)
1017 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -05001018
Qu Wenruod38ed272015-10-12 14:53:37 +08001019 set_state_bits(tree, prealloc, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -04001020 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001021 merge_state(tree, prealloc);
1022 prealloc = NULL;
1023 goto out;
1024 }
1025
David Sterbab5a4ba12016-04-27 01:02:15 +02001026search_again:
1027 if (start > end)
1028 goto out;
1029 spin_unlock(&tree->lock);
1030 if (gfpflags_allow_blocking(mask))
1031 cond_resched();
1032 goto again;
Chris Masond1310b22008-01-24 16:13:08 -05001033
1034out:
Chris Masoncad321a2008-12-17 14:51:42 -05001035 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001036 if (prealloc)
1037 free_extent_state(prealloc);
1038
1039 return err;
1040
Chris Masond1310b22008-01-24 16:13:08 -05001041}
Chris Masond1310b22008-01-24 16:13:08 -05001042
David Sterba41074882013-04-29 13:38:46 +00001043int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001044 unsigned bits, u64 * failed_start,
David Sterba41074882013-04-29 13:38:46 +00001045 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001046{
1047 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001048 cached_state, mask, NULL);
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001049}
1050
1051
Josef Bacik462d6fa2011-09-26 13:56:12 -04001052/**
Liu Bo10983f22012-07-11 15:26:19 +08001053 * convert_extent_bit - convert all bits in a given range from one bit to
1054 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001055 * @tree: the io tree to search
1056 * @start: the start offset in bytes
1057 * @end: the end offset in bytes (inclusive)
1058 * @bits: the bits to set in this range
1059 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001060 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001061 *
1062 * This will go through and set bits for the given range. If any states exist
1063 * already in this range they are set with the given bit and cleared of the
1064 * clear_bits. This is only meant to be used by things that are mergeable, ie
1065 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1066 * boundary bits like LOCK.
David Sterba210aa272016-04-26 23:54:39 +02001067 *
1068 * All allocations are done with GFP_NOFS.
Josef Bacik462d6fa2011-09-26 13:56:12 -04001069 */
1070int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001071 unsigned bits, unsigned clear_bits,
David Sterba210aa272016-04-26 23:54:39 +02001072 struct extent_state **cached_state)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001073{
1074 struct extent_state *state;
1075 struct extent_state *prealloc = NULL;
1076 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001077 struct rb_node **p;
1078 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001079 int err = 0;
1080 u64 last_start;
1081 u64 last_end;
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001082 bool first_iteration = true;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001083
Josef Bacika5dee372013-12-13 10:02:44 -05001084 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +00001085
Josef Bacik462d6fa2011-09-26 13:56:12 -04001086again:
David Sterba210aa272016-04-26 23:54:39 +02001087 if (!prealloc) {
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001088 /*
1089 * Best effort, don't worry if extent state allocation fails
1090 * here for the first iteration. We might have a cached state
1091 * that matches exactly the target range, in which case no
1092 * extent state allocations are needed. We'll only know this
1093 * after locking the tree.
1094 */
David Sterba210aa272016-04-26 23:54:39 +02001095 prealloc = alloc_extent_state(GFP_NOFS);
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001096 if (!prealloc && !first_iteration)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001097 return -ENOMEM;
1098 }
1099
1100 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001101 if (cached_state && *cached_state) {
1102 state = *cached_state;
1103 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +01001104 extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001105 node = &state->rb_node;
1106 goto hit_next;
1107 }
1108 }
1109
Josef Bacik462d6fa2011-09-26 13:56:12 -04001110 /*
1111 * this search will find all the extents that end after
1112 * our range starts.
1113 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001114 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001115 if (!node) {
1116 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001117 if (!prealloc) {
1118 err = -ENOMEM;
1119 goto out;
1120 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001121 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001122 &p, &parent, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001123 if (err)
1124 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001125 cache_state(prealloc, cached_state);
1126 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001127 goto out;
1128 }
1129 state = rb_entry(node, struct extent_state, rb_node);
1130hit_next:
1131 last_start = state->start;
1132 last_end = state->end;
1133
1134 /*
1135 * | ---- desired range ---- |
1136 * | state |
1137 *
1138 * Just lock what we found and keep going
1139 */
1140 if (state->start == start && state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001141 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001142 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001143 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001144 if (last_end == (u64)-1)
1145 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001146 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001147 if (start < end && state && state->start == start &&
1148 !need_resched())
1149 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001150 goto search_again;
1151 }
1152
1153 /*
1154 * | ---- desired range ---- |
1155 * | state |
1156 * or
1157 * | ------------- state -------------- |
1158 *
1159 * We need to split the extent we found, and may flip bits on
1160 * second half.
1161 *
1162 * If the extent we found extends past our
1163 * range, we just split and search again. It'll get split
1164 * again the next time though.
1165 *
1166 * If the extent we found is inside our range, we set the
1167 * desired bit on it.
1168 */
1169 if (state->start < start) {
1170 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001171 if (!prealloc) {
1172 err = -ENOMEM;
1173 goto out;
1174 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001175 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001176 if (err)
1177 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001178 prealloc = NULL;
1179 if (err)
1180 goto out;
1181 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001182 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001183 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001184 state = clear_state_bit(tree, state, &clear_bits, 0,
1185 NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001186 if (last_end == (u64)-1)
1187 goto out;
1188 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001189 if (start < end && state && state->start == start &&
1190 !need_resched())
1191 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001192 }
1193 goto search_again;
1194 }
1195 /*
1196 * | ---- desired range ---- |
1197 * | state | or | state |
1198 *
1199 * There's a hole, we need to insert something in it and
1200 * ignore the extent we found.
1201 */
1202 if (state->start > start) {
1203 u64 this_end;
1204 if (end < last_start)
1205 this_end = end;
1206 else
1207 this_end = last_start - 1;
1208
1209 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001210 if (!prealloc) {
1211 err = -ENOMEM;
1212 goto out;
1213 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001214
1215 /*
1216 * Avoid to free 'prealloc' if it can be merged with
1217 * the later extent.
1218 */
1219 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001220 NULL, NULL, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001221 if (err)
1222 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001223 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001224 prealloc = NULL;
1225 start = this_end + 1;
1226 goto search_again;
1227 }
1228 /*
1229 * | ---- desired range ---- |
1230 * | state |
1231 * We need to split the extent, and set the bit
1232 * on the first half
1233 */
1234 if (state->start <= end && state->end > end) {
1235 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001236 if (!prealloc) {
1237 err = -ENOMEM;
1238 goto out;
1239 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001240
1241 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001242 if (err)
1243 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001244
Qu Wenruod38ed272015-10-12 14:53:37 +08001245 set_state_bits(tree, prealloc, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001246 cache_state(prealloc, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001247 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001248 prealloc = NULL;
1249 goto out;
1250 }
1251
Josef Bacik462d6fa2011-09-26 13:56:12 -04001252search_again:
1253 if (start > end)
1254 goto out;
1255 spin_unlock(&tree->lock);
David Sterba210aa272016-04-26 23:54:39 +02001256 cond_resched();
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001257 first_iteration = false;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001258 goto again;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001259
1260out:
1261 spin_unlock(&tree->lock);
1262 if (prealloc)
1263 free_extent_state(prealloc);
1264
1265 return err;
1266}
1267
Chris Masond1310b22008-01-24 16:13:08 -05001268/* wrappers around set/clear extent bit */
Qu Wenruod38ed272015-10-12 14:53:37 +08001269int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba2c53b912016-04-26 23:54:39 +02001270 unsigned bits, struct extent_changeset *changeset)
Qu Wenruod38ed272015-10-12 14:53:37 +08001271{
1272 /*
1273 * We don't support EXTENT_LOCKED yet, as current changeset will
1274 * record any bits changed, so for EXTENT_LOCKED case, it will
1275 * either fail with -EEXIST or changeset will record the whole
1276 * range.
1277 */
1278 BUG_ON(bits & EXTENT_LOCKED);
1279
David Sterba2c53b912016-04-26 23:54:39 +02001280 return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
Qu Wenruod38ed272015-10-12 14:53:37 +08001281 changeset);
1282}
1283
Qu Wenruofefdc552015-10-12 15:35:38 +08001284int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1285 unsigned bits, int wake, int delete,
David Sterbaae0f1622017-10-31 16:37:52 +01001286 struct extent_state **cached)
Qu Wenruofefdc552015-10-12 15:35:38 +08001287{
1288 return __clear_extent_bit(tree, start, end, bits, wake, delete,
David Sterbaae0f1622017-10-31 16:37:52 +01001289 cached, GFP_NOFS, NULL);
Qu Wenruofefdc552015-10-12 15:35:38 +08001290}
1291
Qu Wenruofefdc552015-10-12 15:35:38 +08001292int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaf734c442016-04-26 23:54:39 +02001293 unsigned bits, struct extent_changeset *changeset)
Qu Wenruofefdc552015-10-12 15:35:38 +08001294{
1295 /*
1296 * Don't support EXTENT_LOCKED case, same reason as
1297 * set_record_extent_bits().
1298 */
1299 BUG_ON(bits & EXTENT_LOCKED);
1300
David Sterbaf734c442016-04-26 23:54:39 +02001301 return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
Qu Wenruofefdc552015-10-12 15:35:38 +08001302 changeset);
1303}
1304
Chris Masond352ac62008-09-29 15:18:18 -04001305/*
1306 * either insert or lock state struct between start and end use mask to tell
1307 * us if waiting is desired.
1308 */
Chris Mason1edbb732009-09-02 13:24:36 -04001309int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaff13db42015-12-03 14:30:40 +01001310 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001311{
1312 int err;
1313 u64 failed_start;
David Sterba9ee49a042015-01-14 19:52:13 +01001314
Chris Masond1310b22008-01-24 16:13:08 -05001315 while (1) {
David Sterbaff13db42015-12-03 14:30:40 +01001316 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001317 EXTENT_LOCKED, &failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001318 cached_state, GFP_NOFS, NULL);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001319 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001320 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1321 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001322 } else
Chris Masond1310b22008-01-24 16:13:08 -05001323 break;
Chris Masond1310b22008-01-24 16:13:08 -05001324 WARN_ON(start > end);
1325 }
1326 return err;
1327}
Chris Masond1310b22008-01-24 16:13:08 -05001328
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001329int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001330{
1331 int err;
1332 u64 failed_start;
1333
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001334 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
Qu Wenruod38ed272015-10-12 14:53:37 +08001335 &failed_start, NULL, GFP_NOFS, NULL);
Yan Zheng66435582008-10-30 14:19:50 -04001336 if (err == -EEXIST) {
1337 if (failed_start > start)
1338 clear_extent_bit(tree, start, failed_start - 1,
David Sterbaae0f1622017-10-31 16:37:52 +01001339 EXTENT_LOCKED, 1, 0, NULL);
Josef Bacik25179202008-10-29 14:49:05 -04001340 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001341 }
Josef Bacik25179202008-10-29 14:49:05 -04001342 return 1;
1343}
Josef Bacik25179202008-10-29 14:49:05 -04001344
David Sterbabd1fa4f2015-12-03 13:08:59 +01001345void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001346{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001347 unsigned long index = start >> PAGE_SHIFT;
1348 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001349 struct page *page;
1350
1351 while (index <= end_index) {
1352 page = find_get_page(inode->i_mapping, index);
1353 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1354 clear_page_dirty_for_io(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001355 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001356 index++;
1357 }
Chris Mason4adaa612013-03-26 13:07:00 -04001358}
1359
David Sterbaf6311572015-12-03 13:08:59 +01001360void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001361{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001362 unsigned long index = start >> PAGE_SHIFT;
1363 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001364 struct page *page;
1365
1366 while (index <= end_index) {
1367 page = find_get_page(inode->i_mapping, index);
1368 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Mason4adaa612013-03-26 13:07:00 -04001369 __set_page_dirty_nobuffers(page);
Konstantin Khebnikov8d386332015-02-11 15:26:55 -08001370 account_page_redirty(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001371 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001372 index++;
1373 }
Chris Mason4adaa612013-03-26 13:07:00 -04001374}
1375
Chris Masond352ac62008-09-29 15:18:18 -04001376/* find the first state struct with 'bits' set after 'start', and
1377 * return it. tree->lock must be held. NULL will returned if
1378 * nothing was found after 'start'
1379 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001380static struct extent_state *
1381find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +01001382 u64 start, unsigned bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001383{
1384 struct rb_node *node;
1385 struct extent_state *state;
1386
1387 /*
1388 * this search will find all the extents that end after
1389 * our range starts.
1390 */
1391 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001392 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001393 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001394
Chris Masond3977122009-01-05 21:25:51 -05001395 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001396 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001397 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001398 return state;
Chris Masond3977122009-01-05 21:25:51 -05001399
Chris Masond7fc6402008-02-18 12:12:38 -05001400 node = rb_next(node);
1401 if (!node)
1402 break;
1403 }
1404out:
1405 return NULL;
1406}
Chris Masond7fc6402008-02-18 12:12:38 -05001407
Chris Masond352ac62008-09-29 15:18:18 -04001408/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001409 * find the first offset in the io tree with 'bits' set. zero is
1410 * returned if we find something, and *start_ret and *end_ret are
1411 * set to reflect the state struct that was found.
1412 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001413 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001414 */
1415int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba9ee49a042015-01-14 19:52:13 +01001416 u64 *start_ret, u64 *end_ret, unsigned bits,
Josef Bacike6138872012-09-27 17:07:30 -04001417 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001418{
1419 struct extent_state *state;
1420 int ret = 1;
1421
1422 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001423 if (cached_state && *cached_state) {
1424 state = *cached_state;
Filipe Manana27a35072014-07-06 20:09:59 +01001425 if (state->end == start - 1 && extent_state_in_tree(state)) {
Liu Bo9688e9a2018-08-23 03:14:53 +08001426 while ((state = next_state(state)) != NULL) {
Josef Bacike6138872012-09-27 17:07:30 -04001427 if (state->state & bits)
1428 goto got_it;
Josef Bacike6138872012-09-27 17:07:30 -04001429 }
1430 free_extent_state(*cached_state);
1431 *cached_state = NULL;
1432 goto out;
1433 }
1434 free_extent_state(*cached_state);
1435 *cached_state = NULL;
1436 }
1437
Xiao Guangrong69261c42011-07-14 03:19:45 +00001438 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001439got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001440 if (state) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +01001441 cache_state_if_flags(state, cached_state, 0);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001442 *start_ret = state->start;
1443 *end_ret = state->end;
1444 ret = 0;
1445 }
Josef Bacike6138872012-09-27 17:07:30 -04001446out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001447 spin_unlock(&tree->lock);
1448 return ret;
1449}
1450
1451/*
Chris Masond352ac62008-09-29 15:18:18 -04001452 * find a contiguous range of bytes in the file marked as delalloc, not
1453 * more than 'max_bytes'. start and end are used to return the range,
1454 *
Lu Fengqi3522e902018-11-29 11:33:38 +08001455 * true is returned if we find something, false if nothing was in the tree
Chris Masond352ac62008-09-29 15:18:18 -04001456 */
Lu Fengqi3522e902018-11-29 11:33:38 +08001457static noinline bool find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001458 u64 *start, u64 *end, u64 max_bytes,
1459 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001460{
1461 struct rb_node *node;
1462 struct extent_state *state;
1463 u64 cur_start = *start;
Lu Fengqi3522e902018-11-29 11:33:38 +08001464 bool found = false;
Chris Masond1310b22008-01-24 16:13:08 -05001465 u64 total_bytes = 0;
1466
Chris Masoncad321a2008-12-17 14:51:42 -05001467 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001468
Chris Masond1310b22008-01-24 16:13:08 -05001469 /*
1470 * this search will find all the extents that end after
1471 * our range starts.
1472 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001473 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001474 if (!node) {
Lu Fengqi3522e902018-11-29 11:33:38 +08001475 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001476 goto out;
1477 }
1478
Chris Masond3977122009-01-05 21:25:51 -05001479 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001480 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001481 if (found && (state->start != cur_start ||
1482 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001483 goto out;
1484 }
1485 if (!(state->state & EXTENT_DELALLOC)) {
1486 if (!found)
1487 *end = state->end;
1488 goto out;
1489 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001490 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001491 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001492 *cached_state = state;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +02001493 refcount_inc(&state->refs);
Josef Bacikc2a128d2010-02-02 21:19:11 +00001494 }
Lu Fengqi3522e902018-11-29 11:33:38 +08001495 found = true;
Chris Masond1310b22008-01-24 16:13:08 -05001496 *end = state->end;
1497 cur_start = state->end + 1;
1498 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001499 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001500 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001501 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001502 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001503 break;
1504 }
1505out:
Chris Masoncad321a2008-12-17 14:51:42 -05001506 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001507 return found;
1508}
1509
Liu Boda2c7002017-02-10 16:41:05 +01001510static int __process_pages_contig(struct address_space *mapping,
1511 struct page *locked_page,
1512 pgoff_t start_index, pgoff_t end_index,
1513 unsigned long page_ops, pgoff_t *index_ret);
1514
Jeff Mahoney143bede2012-03-01 14:56:26 +01001515static noinline void __unlock_for_delalloc(struct inode *inode,
1516 struct page *locked_page,
1517 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001518{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001519 unsigned long index = start >> PAGE_SHIFT;
1520 unsigned long end_index = end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001521
Liu Bo76c00212017-02-10 16:42:14 +01001522 ASSERT(locked_page);
Chris Masonc8b97812008-10-29 14:49:59 -04001523 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001524 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001525
Liu Bo76c00212017-02-10 16:42:14 +01001526 __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1527 PAGE_UNLOCK, NULL);
Chris Masonc8b97812008-10-29 14:49:59 -04001528}
1529
1530static noinline int lock_delalloc_pages(struct inode *inode,
1531 struct page *locked_page,
1532 u64 delalloc_start,
1533 u64 delalloc_end)
1534{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001535 unsigned long index = delalloc_start >> PAGE_SHIFT;
Liu Bo76c00212017-02-10 16:42:14 +01001536 unsigned long index_ret = index;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001537 unsigned long end_index = delalloc_end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001538 int ret;
Chris Masonc8b97812008-10-29 14:49:59 -04001539
Liu Bo76c00212017-02-10 16:42:14 +01001540 ASSERT(locked_page);
Chris Masonc8b97812008-10-29 14:49:59 -04001541 if (index == locked_page->index && index == end_index)
1542 return 0;
1543
Liu Bo76c00212017-02-10 16:42:14 +01001544 ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1545 end_index, PAGE_LOCK, &index_ret);
1546 if (ret == -EAGAIN)
1547 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1548 (u64)index_ret << PAGE_SHIFT);
Chris Masonc8b97812008-10-29 14:49:59 -04001549 return ret;
1550}
1551
1552/*
Lu Fengqi3522e902018-11-29 11:33:38 +08001553 * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1554 * more than @max_bytes. @Start and @end are used to return the range,
Chris Masonc8b97812008-10-29 14:49:59 -04001555 *
Lu Fengqi3522e902018-11-29 11:33:38 +08001556 * Return: true if we find something
1557 * false if nothing was in the tree
Chris Masonc8b97812008-10-29 14:49:59 -04001558 */
Johannes Thumshirnce9f9672018-11-19 10:38:17 +01001559EXPORT_FOR_TESTS
Lu Fengqi3522e902018-11-29 11:33:38 +08001560noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
Josef Bacik294e30f2013-10-09 12:00:56 -04001561 struct extent_io_tree *tree,
1562 struct page *locked_page, u64 *start,
Nikolay Borisov917aace2018-10-26 14:43:20 +03001563 u64 *end)
Chris Masonc8b97812008-10-29 14:49:59 -04001564{
Nikolay Borisov917aace2018-10-26 14:43:20 +03001565 u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001566 u64 delalloc_start;
1567 u64 delalloc_end;
Lu Fengqi3522e902018-11-29 11:33:38 +08001568 bool found;
Chris Mason9655d292009-09-02 15:22:30 -04001569 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001570 int ret;
1571 int loops = 0;
1572
1573again:
1574 /* step one, find a bunch of delalloc bytes starting at start */
1575 delalloc_start = *start;
1576 delalloc_end = 0;
1577 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001578 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001579 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001580 *start = delalloc_start;
1581 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001582 free_extent_state(cached_state);
Lu Fengqi3522e902018-11-29 11:33:38 +08001583 return false;
Chris Masonc8b97812008-10-29 14:49:59 -04001584 }
1585
1586 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001587 * start comes from the offset of locked_page. We have to lock
1588 * pages in order, so we can't process delalloc bytes before
1589 * locked_page
1590 */
Chris Masond3977122009-01-05 21:25:51 -05001591 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001592 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001593
1594 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001595 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001596 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001597 if (delalloc_end + 1 - delalloc_start > max_bytes)
1598 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001599
Chris Masonc8b97812008-10-29 14:49:59 -04001600 /* step two, lock all the pages after the page that has start */
1601 ret = lock_delalloc_pages(inode, locked_page,
1602 delalloc_start, delalloc_end);
Nikolay Borisov9bfd61d2018-10-26 14:43:21 +03001603 ASSERT(!ret || ret == -EAGAIN);
Chris Masonc8b97812008-10-29 14:49:59 -04001604 if (ret == -EAGAIN) {
1605 /* some of the pages are gone, lets avoid looping by
1606 * shortening the size of the delalloc range we're searching
1607 */
Chris Mason9655d292009-09-02 15:22:30 -04001608 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001609 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001610 if (!loops) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001611 max_bytes = PAGE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001612 loops = 1;
1613 goto again;
1614 } else {
Lu Fengqi3522e902018-11-29 11:33:38 +08001615 found = false;
Chris Masonc8b97812008-10-29 14:49:59 -04001616 goto out_failed;
1617 }
1618 }
Chris Masonc8b97812008-10-29 14:49:59 -04001619
1620 /* step three, lock the state bits for the whole range */
David Sterbaff13db42015-12-03 14:30:40 +01001621 lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001622
1623 /* then test to make sure it is all still delalloc */
1624 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001625 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001626 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001627 unlock_extent_cached(tree, delalloc_start, delalloc_end,
David Sterbae43bbe52017-12-12 21:43:52 +01001628 &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001629 __unlock_for_delalloc(inode, locked_page,
1630 delalloc_start, delalloc_end);
1631 cond_resched();
1632 goto again;
1633 }
Chris Mason9655d292009-09-02 15:22:30 -04001634 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001635 *start = delalloc_start;
1636 *end = delalloc_end;
1637out_failed:
1638 return found;
1639}
1640
Liu Boda2c7002017-02-10 16:41:05 +01001641static int __process_pages_contig(struct address_space *mapping,
1642 struct page *locked_page,
1643 pgoff_t start_index, pgoff_t end_index,
1644 unsigned long page_ops, pgoff_t *index_ret)
Chris Masonc8b97812008-10-29 14:49:59 -04001645{
Liu Bo873695b2017-02-02 17:49:22 -08001646 unsigned long nr_pages = end_index - start_index + 1;
Liu Boda2c7002017-02-10 16:41:05 +01001647 unsigned long pages_locked = 0;
Liu Bo873695b2017-02-02 17:49:22 -08001648 pgoff_t index = start_index;
Chris Masonc8b97812008-10-29 14:49:59 -04001649 struct page *pages[16];
Liu Bo873695b2017-02-02 17:49:22 -08001650 unsigned ret;
Liu Boda2c7002017-02-10 16:41:05 +01001651 int err = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001652 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001653
Liu Boda2c7002017-02-10 16:41:05 +01001654 if (page_ops & PAGE_LOCK) {
1655 ASSERT(page_ops == PAGE_LOCK);
1656 ASSERT(index_ret && *index_ret == start_index);
1657 }
1658
Filipe Manana704de492014-10-06 22:14:22 +01001659 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
Liu Bo873695b2017-02-02 17:49:22 -08001660 mapping_set_error(mapping, -EIO);
Filipe Manana704de492014-10-06 22:14:22 +01001661
Chris Masond3977122009-01-05 21:25:51 -05001662 while (nr_pages > 0) {
Liu Bo873695b2017-02-02 17:49:22 -08001663 ret = find_get_pages_contig(mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001664 min_t(unsigned long,
1665 nr_pages, ARRAY_SIZE(pages)), pages);
Liu Boda2c7002017-02-10 16:41:05 +01001666 if (ret == 0) {
1667 /*
1668 * Only if we're going to lock these pages,
1669 * can we find nothing at @index.
1670 */
1671 ASSERT(page_ops & PAGE_LOCK);
Liu Bo49d4a332017-03-06 18:20:56 -08001672 err = -EAGAIN;
1673 goto out;
Liu Boda2c7002017-02-10 16:41:05 +01001674 }
Chris Mason8b62b722009-09-02 16:53:46 -04001675
Liu Boda2c7002017-02-10 16:41:05 +01001676 for (i = 0; i < ret; i++) {
Josef Bacikc2790a22013-07-29 11:20:47 -04001677 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001678 SetPagePrivate2(pages[i]);
1679
Chris Masonc8b97812008-10-29 14:49:59 -04001680 if (pages[i] == locked_page) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001681 put_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001682 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001683 continue;
1684 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001685 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001686 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001687 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001688 set_page_writeback(pages[i]);
Filipe Manana704de492014-10-06 22:14:22 +01001689 if (page_ops & PAGE_SET_ERROR)
1690 SetPageError(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001691 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001692 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001693 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001694 unlock_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001695 if (page_ops & PAGE_LOCK) {
1696 lock_page(pages[i]);
1697 if (!PageDirty(pages[i]) ||
1698 pages[i]->mapping != mapping) {
1699 unlock_page(pages[i]);
1700 put_page(pages[i]);
1701 err = -EAGAIN;
1702 goto out;
1703 }
1704 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001705 put_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001706 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001707 }
1708 nr_pages -= ret;
1709 index += ret;
1710 cond_resched();
1711 }
Liu Boda2c7002017-02-10 16:41:05 +01001712out:
1713 if (err && index_ret)
1714 *index_ret = start_index + pages_locked - 1;
1715 return err;
Chris Masonc8b97812008-10-29 14:49:59 -04001716}
Chris Masonc8b97812008-10-29 14:49:59 -04001717
Liu Bo873695b2017-02-02 17:49:22 -08001718void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1719 u64 delalloc_end, struct page *locked_page,
1720 unsigned clear_bits,
1721 unsigned long page_ops)
1722{
1723 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
David Sterbaae0f1622017-10-31 16:37:52 +01001724 NULL);
Liu Bo873695b2017-02-02 17:49:22 -08001725
1726 __process_pages_contig(inode->i_mapping, locked_page,
1727 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
Liu Boda2c7002017-02-10 16:41:05 +01001728 page_ops, NULL);
Liu Bo873695b2017-02-02 17:49:22 -08001729}
1730
Chris Masond352ac62008-09-29 15:18:18 -04001731/*
1732 * count the number of bytes in the tree that have a given bit(s)
1733 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1734 * cached. The total number found is returned.
1735 */
Chris Masond1310b22008-01-24 16:13:08 -05001736u64 count_range_bits(struct extent_io_tree *tree,
1737 u64 *start, u64 search_end, u64 max_bytes,
David Sterba9ee49a042015-01-14 19:52:13 +01001738 unsigned bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001739{
1740 struct rb_node *node;
1741 struct extent_state *state;
1742 u64 cur_start = *start;
1743 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001744 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001745 int found = 0;
1746
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301747 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05001748 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05001749
Chris Masoncad321a2008-12-17 14:51:42 -05001750 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001751 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1752 total_bytes = tree->dirty_bytes;
1753 goto out;
1754 }
1755 /*
1756 * this search will find all the extents that end after
1757 * our range starts.
1758 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001759 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001760 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001761 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001762
Chris Masond3977122009-01-05 21:25:51 -05001763 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001764 state = rb_entry(node, struct extent_state, rb_node);
1765 if (state->start > search_end)
1766 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001767 if (contig && found && state->start > last + 1)
1768 break;
1769 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001770 total_bytes += min(search_end, state->end) + 1 -
1771 max(cur_start, state->start);
1772 if (total_bytes >= max_bytes)
1773 break;
1774 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001775 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001776 found = 1;
1777 }
Chris Masonec29ed52011-02-23 16:23:20 -05001778 last = state->end;
1779 } else if (contig && found) {
1780 break;
Chris Masond1310b22008-01-24 16:13:08 -05001781 }
1782 node = rb_next(node);
1783 if (!node)
1784 break;
1785 }
1786out:
Chris Masoncad321a2008-12-17 14:51:42 -05001787 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001788 return total_bytes;
1789}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001790
Chris Masond352ac62008-09-29 15:18:18 -04001791/*
1792 * set the private field for a given byte offset in the tree. If there isn't
1793 * an extent_state there already, this does nothing.
1794 */
Arnd Bergmannf827ba92016-02-22 22:53:20 +01001795static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
David Sterba47dc1962016-02-11 13:24:13 +01001796 struct io_failure_record *failrec)
Chris Masond1310b22008-01-24 16:13:08 -05001797{
1798 struct rb_node *node;
1799 struct extent_state *state;
1800 int ret = 0;
1801
Chris Masoncad321a2008-12-17 14:51:42 -05001802 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001803 /*
1804 * this search will find all the extents that end after
1805 * our range starts.
1806 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001807 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001808 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001809 ret = -ENOENT;
1810 goto out;
1811 }
1812 state = rb_entry(node, struct extent_state, rb_node);
1813 if (state->start != start) {
1814 ret = -ENOENT;
1815 goto out;
1816 }
David Sterba47dc1962016-02-11 13:24:13 +01001817 state->failrec = failrec;
Chris Masond1310b22008-01-24 16:13:08 -05001818out:
Chris Masoncad321a2008-12-17 14:51:42 -05001819 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001820 return ret;
1821}
1822
Arnd Bergmannf827ba92016-02-22 22:53:20 +01001823static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
David Sterba47dc1962016-02-11 13:24:13 +01001824 struct io_failure_record **failrec)
Chris Masond1310b22008-01-24 16:13:08 -05001825{
1826 struct rb_node *node;
1827 struct extent_state *state;
1828 int ret = 0;
1829
Chris Masoncad321a2008-12-17 14:51:42 -05001830 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001831 /*
1832 * this search will find all the extents that end after
1833 * our range starts.
1834 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001835 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001836 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001837 ret = -ENOENT;
1838 goto out;
1839 }
1840 state = rb_entry(node, struct extent_state, rb_node);
1841 if (state->start != start) {
1842 ret = -ENOENT;
1843 goto out;
1844 }
David Sterba47dc1962016-02-11 13:24:13 +01001845 *failrec = state->failrec;
Chris Masond1310b22008-01-24 16:13:08 -05001846out:
Chris Masoncad321a2008-12-17 14:51:42 -05001847 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001848 return ret;
1849}
1850
1851/*
1852 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001853 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001854 * has the bits set. Otherwise, 1 is returned if any bit in the
1855 * range is found set.
1856 */
1857int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001858 unsigned bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001859{
1860 struct extent_state *state = NULL;
1861 struct rb_node *node;
1862 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001863
Chris Masoncad321a2008-12-17 14:51:42 -05001864 spin_lock(&tree->lock);
Filipe Manana27a35072014-07-06 20:09:59 +01001865 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001866 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001867 node = &cached->rb_node;
1868 else
1869 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001870 while (node && start <= end) {
1871 state = rb_entry(node, struct extent_state, rb_node);
1872
1873 if (filled && state->start > start) {
1874 bitset = 0;
1875 break;
1876 }
1877
1878 if (state->start > end)
1879 break;
1880
1881 if (state->state & bits) {
1882 bitset = 1;
1883 if (!filled)
1884 break;
1885 } else if (filled) {
1886 bitset = 0;
1887 break;
1888 }
Chris Mason46562ce2009-09-23 20:23:16 -04001889
1890 if (state->end == (u64)-1)
1891 break;
1892
Chris Masond1310b22008-01-24 16:13:08 -05001893 start = state->end + 1;
1894 if (start > end)
1895 break;
1896 node = rb_next(node);
1897 if (!node) {
1898 if (filled)
1899 bitset = 0;
1900 break;
1901 }
1902 }
Chris Masoncad321a2008-12-17 14:51:42 -05001903 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001904 return bitset;
1905}
Chris Masond1310b22008-01-24 16:13:08 -05001906
1907/*
1908 * helper function to set a given page up to date if all the
1909 * extents in the tree for that page are up to date
1910 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001911static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001912{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001913 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001914 u64 end = start + PAGE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001915 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001916 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001917}
1918
Josef Bacik7870d082017-05-05 11:57:15 -04001919int free_io_failure(struct extent_io_tree *failure_tree,
1920 struct extent_io_tree *io_tree,
1921 struct io_failure_record *rec)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001922{
1923 int ret;
1924 int err = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001925
David Sterba47dc1962016-02-11 13:24:13 +01001926 set_state_failrec(failure_tree, rec->start, NULL);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001927 ret = clear_extent_bits(failure_tree, rec->start,
1928 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02001929 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001930 if (ret)
1931 err = ret;
1932
Josef Bacik7870d082017-05-05 11:57:15 -04001933 ret = clear_extent_bits(io_tree, rec->start,
David Woodhouse53b381b2013-01-29 18:40:14 -05001934 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02001935 EXTENT_DAMAGED);
David Woodhouse53b381b2013-01-29 18:40:14 -05001936 if (ret && !err)
1937 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001938
1939 kfree(rec);
1940 return err;
1941}
1942
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001943/*
1944 * this bypasses the standard btrfs submit functions deliberately, as
1945 * the standard behavior is to write all copies in a raid setup. here we only
1946 * want to write the one bad copy. so we do the mapping for ourselves and issue
1947 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01001948 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001949 * actually prevents the read that triggered the error from finishing.
1950 * currently, there can be no more than two copies of every data bit. thus,
1951 * exactly one rewrite is required.
1952 */
Josef Bacik6ec656b2017-05-05 11:57:14 -04001953int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
1954 u64 length, u64 logical, struct page *page,
1955 unsigned int pg_offset, int mirror_num)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001956{
1957 struct bio *bio;
1958 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001959 u64 map_length = 0;
1960 u64 sector;
1961 struct btrfs_bio *bbio = NULL;
1962 int ret;
1963
Linus Torvalds1751e8a2017-11-27 13:05:09 -08001964 ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001965 BUG_ON(!mirror_num);
1966
David Sterbac5e4c3d2017-06-12 17:29:41 +02001967 bio = btrfs_io_bio_alloc(1);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001968 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001969 map_length = length;
1970
Filipe Mananab5de8d02016-05-27 22:21:27 +01001971 /*
1972 * Avoid races with device replace and make sure our bbio has devices
1973 * associated to its stripes that don't go away while we are doing the
1974 * read repair operation.
1975 */
1976 btrfs_bio_counter_inc_blocked(fs_info);
Nikolay Borisove4ff5fb2017-07-19 10:48:42 +03001977 if (btrfs_is_parity_mirror(fs_info, logical, length)) {
Liu Boc7253282017-03-29 10:53:58 -07001978 /*
1979 * Note that we don't use BTRFS_MAP_WRITE because it's supposed
1980 * to update all raid stripes, but here we just want to correct
1981 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
1982 * stripe's dev and sector.
1983 */
1984 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
1985 &map_length, &bbio, 0);
1986 if (ret) {
1987 btrfs_bio_counter_dec(fs_info);
1988 bio_put(bio);
1989 return -EIO;
1990 }
1991 ASSERT(bbio->mirror_num == 1);
1992 } else {
1993 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
1994 &map_length, &bbio, mirror_num);
1995 if (ret) {
1996 btrfs_bio_counter_dec(fs_info);
1997 bio_put(bio);
1998 return -EIO;
1999 }
2000 BUG_ON(mirror_num != bbio->mirror_num);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002001 }
Liu Boc7253282017-03-29 10:53:58 -07002002
2003 sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002004 bio->bi_iter.bi_sector = sector;
Liu Boc7253282017-03-29 10:53:58 -07002005 dev = bbio->stripes[bbio->mirror_num - 1].dev;
Zhao Lei6e9606d2015-01-20 15:11:34 +08002006 btrfs_put_bbio(bbio);
Anand Jainebbede42017-12-04 12:54:52 +08002007 if (!dev || !dev->bdev ||
2008 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
Filipe Mananab5de8d02016-05-27 22:21:27 +01002009 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002010 bio_put(bio);
2011 return -EIO;
2012 }
Christoph Hellwig74d46992017-08-23 19:10:32 +02002013 bio_set_dev(bio, dev->bdev);
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002014 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
Miao Xieffdd2012014-09-12 18:44:00 +08002015 bio_add_page(bio, page, length, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002016
Mike Christie4e49ea42016-06-05 14:31:41 -05002017 if (btrfsic_submit_bio_wait(bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002018 /* try to remap that extent elsewhere? */
Filipe Mananab5de8d02016-05-27 22:21:27 +01002019 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002020 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002021 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002022 return -EIO;
2023 }
2024
David Sterbab14af3b2015-10-08 10:43:10 +02002025 btrfs_info_rl_in_rcu(fs_info,
2026 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
Josef Bacik6ec656b2017-05-05 11:57:14 -04002027 ino, start,
Miao Xie1203b682014-09-12 18:44:01 +08002028 rcu_str_deref(dev->name), sector);
Filipe Mananab5de8d02016-05-27 22:21:27 +01002029 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002030 bio_put(bio);
2031 return 0;
2032}
2033
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002034int repair_eb_io_failure(struct btrfs_fs_info *fs_info,
2035 struct extent_buffer *eb, int mirror_num)
Josef Bacikea466792012-03-26 21:57:36 -04002036{
Josef Bacikea466792012-03-26 21:57:36 -04002037 u64 start = eb->start;
David Sterbacc5e31a2018-03-01 18:20:27 +01002038 int i, num_pages = num_extent_pages(eb);
Chris Masond95603b2012-04-12 15:55:15 -04002039 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002040
David Howellsbc98a422017-07-17 08:45:34 +01002041 if (sb_rdonly(fs_info->sb))
Ilya Dryomov908960c2013-11-03 19:06:39 +02002042 return -EROFS;
2043
Josef Bacikea466792012-03-26 21:57:36 -04002044 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02002045 struct page *p = eb->pages[i];
Miao Xie1203b682014-09-12 18:44:01 +08002046
Josef Bacik6ec656b2017-05-05 11:57:14 -04002047 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
Miao Xie1203b682014-09-12 18:44:01 +08002048 start - page_offset(p), mirror_num);
Josef Bacikea466792012-03-26 21:57:36 -04002049 if (ret)
2050 break;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002051 start += PAGE_SIZE;
Josef Bacikea466792012-03-26 21:57:36 -04002052 }
2053
2054 return ret;
2055}
2056
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002057/*
2058 * each time an IO finishes, we do a fast check in the IO failure tree
2059 * to see if we need to process or clean up an io_failure_record
2060 */
Josef Bacik7870d082017-05-05 11:57:15 -04002061int clean_io_failure(struct btrfs_fs_info *fs_info,
2062 struct extent_io_tree *failure_tree,
2063 struct extent_io_tree *io_tree, u64 start,
2064 struct page *page, u64 ino, unsigned int pg_offset)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002065{
2066 u64 private;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002067 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002068 struct extent_state *state;
2069 int num_copies;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002070 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002071
2072 private = 0;
Josef Bacik7870d082017-05-05 11:57:15 -04002073 ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2074 EXTENT_DIRTY, 0);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002075 if (!ret)
2076 return 0;
2077
Josef Bacik7870d082017-05-05 11:57:15 -04002078 ret = get_state_failrec(failure_tree, start, &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002079 if (ret)
2080 return 0;
2081
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002082 BUG_ON(!failrec->this_mirror);
2083
2084 if (failrec->in_validation) {
2085 /* there was no real error, just free the record */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002086 btrfs_debug(fs_info,
2087 "clean_io_failure: freeing dummy error at %llu",
2088 failrec->start);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002089 goto out;
2090 }
David Howellsbc98a422017-07-17 08:45:34 +01002091 if (sb_rdonly(fs_info->sb))
Ilya Dryomov908960c2013-11-03 19:06:39 +02002092 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002093
Josef Bacik7870d082017-05-05 11:57:15 -04002094 spin_lock(&io_tree->lock);
2095 state = find_first_extent_bit_state(io_tree,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002096 failrec->start,
2097 EXTENT_LOCKED);
Josef Bacik7870d082017-05-05 11:57:15 -04002098 spin_unlock(&io_tree->lock);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002099
Miao Xie883d0de2013-07-25 19:22:35 +08002100 if (state && state->start <= failrec->start &&
2101 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002102 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2103 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002104 if (num_copies > 1) {
Josef Bacik7870d082017-05-05 11:57:15 -04002105 repair_io_failure(fs_info, ino, start, failrec->len,
2106 failrec->logical, page, pg_offset,
2107 failrec->failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002108 }
2109 }
2110
2111out:
Josef Bacik7870d082017-05-05 11:57:15 -04002112 free_io_failure(failure_tree, io_tree, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002113
Miao Xie454ff3d2014-09-12 18:43:58 +08002114 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002115}
2116
Miao Xief6124962014-09-12 18:44:04 +08002117/*
2118 * Can be called when
2119 * - hold extent lock
2120 * - under ordered extent
2121 * - the inode is freeing
2122 */
Nikolay Borisov7ab79562017-02-20 13:50:57 +02002123void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
Miao Xief6124962014-09-12 18:44:04 +08002124{
Nikolay Borisov7ab79562017-02-20 13:50:57 +02002125 struct extent_io_tree *failure_tree = &inode->io_failure_tree;
Miao Xief6124962014-09-12 18:44:04 +08002126 struct io_failure_record *failrec;
2127 struct extent_state *state, *next;
2128
2129 if (RB_EMPTY_ROOT(&failure_tree->state))
2130 return;
2131
2132 spin_lock(&failure_tree->lock);
2133 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2134 while (state) {
2135 if (state->start > end)
2136 break;
2137
2138 ASSERT(state->end <= end);
2139
2140 next = next_state(state);
2141
David Sterba47dc1962016-02-11 13:24:13 +01002142 failrec = state->failrec;
Miao Xief6124962014-09-12 18:44:04 +08002143 free_extent_state(state);
2144 kfree(failrec);
2145
2146 state = next;
2147 }
2148 spin_unlock(&failure_tree->lock);
2149}
2150
Miao Xie2fe63032014-09-12 18:43:59 +08002151int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
David Sterba47dc1962016-02-11 13:24:13 +01002152 struct io_failure_record **failrec_ret)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002153{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002154 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002155 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002156 struct extent_map *em;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002157 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2158 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2159 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002160 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002161 u64 logical;
2162
David Sterba47dc1962016-02-11 13:24:13 +01002163 ret = get_state_failrec(failure_tree, start, &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002164 if (ret) {
2165 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2166 if (!failrec)
2167 return -ENOMEM;
Miao Xie2fe63032014-09-12 18:43:59 +08002168
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002169 failrec->start = start;
2170 failrec->len = end - start + 1;
2171 failrec->this_mirror = 0;
2172 failrec->bio_flags = 0;
2173 failrec->in_validation = 0;
2174
2175 read_lock(&em_tree->lock);
2176 em = lookup_extent_mapping(em_tree, start, failrec->len);
2177 if (!em) {
2178 read_unlock(&em_tree->lock);
2179 kfree(failrec);
2180 return -EIO;
2181 }
2182
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002183 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002184 free_extent_map(em);
2185 em = NULL;
2186 }
2187 read_unlock(&em_tree->lock);
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002188 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002189 kfree(failrec);
2190 return -EIO;
2191 }
Miao Xie2fe63032014-09-12 18:43:59 +08002192
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002193 logical = start - em->start;
2194 logical = em->block_start + logical;
2195 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2196 logical = em->block_start;
2197 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2198 extent_set_compress_type(&failrec->bio_flags,
2199 em->compress_type);
2200 }
Miao Xie2fe63032014-09-12 18:43:59 +08002201
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002202 btrfs_debug(fs_info,
2203 "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2204 logical, start, failrec->len);
Miao Xie2fe63032014-09-12 18:43:59 +08002205
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002206 failrec->logical = logical;
2207 free_extent_map(em);
2208
2209 /* set the bits in the private failure tree */
2210 ret = set_extent_bits(failure_tree, start, end,
David Sterbaceeb0ae2016-04-26 23:54:39 +02002211 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002212 if (ret >= 0)
David Sterba47dc1962016-02-11 13:24:13 +01002213 ret = set_state_failrec(failure_tree, start, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002214 /* set the bits in the inode's tree */
2215 if (ret >= 0)
David Sterbaceeb0ae2016-04-26 23:54:39 +02002216 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002217 if (ret < 0) {
2218 kfree(failrec);
2219 return ret;
2220 }
2221 } else {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002222 btrfs_debug(fs_info,
2223 "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2224 failrec->logical, failrec->start, failrec->len,
2225 failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002226 /*
2227 * when data can be on disk more than twice, add to failrec here
2228 * (e.g. with a list for failed_mirror) to make
2229 * clean_io_failure() clean all those errors at once.
2230 */
2231 }
Miao Xie2fe63032014-09-12 18:43:59 +08002232
2233 *failrec_ret = failrec;
2234
2235 return 0;
2236}
2237
Ming Leia0b60d72017-12-18 20:22:11 +08002238bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
Miao Xie2fe63032014-09-12 18:43:59 +08002239 struct io_failure_record *failrec, int failed_mirror)
2240{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002241 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002242 int num_copies;
2243
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002244 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002245 if (num_copies == 1) {
2246 /*
2247 * we only have a single copy of the data, so don't bother with
2248 * all the retry and error correction code that follows. no
2249 * matter what the error is, it is very likely to persist.
2250 */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002251 btrfs_debug(fs_info,
2252 "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2253 num_copies, failrec->this_mirror, failed_mirror);
Liu Boc3cfb652017-07-13 15:00:50 -07002254 return false;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002255 }
2256
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002257 /*
2258 * there are two premises:
2259 * a) deliver good data to the caller
2260 * b) correct the bad sectors on disk
2261 */
Ming Leia0b60d72017-12-18 20:22:11 +08002262 if (failed_bio_pages > 1) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002263 /*
2264 * to fulfill b), we need to know the exact failing sectors, as
2265 * we don't want to rewrite any more than the failed ones. thus,
2266 * we need separate read requests for the failed bio
2267 *
2268 * if the following BUG_ON triggers, our validation request got
2269 * merged. we need separate requests for our algorithm to work.
2270 */
2271 BUG_ON(failrec->in_validation);
2272 failrec->in_validation = 1;
2273 failrec->this_mirror = failed_mirror;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002274 } else {
2275 /*
2276 * we're ready to fulfill a) and b) alongside. get a good copy
2277 * of the failed sector and if we succeed, we have setup
2278 * everything for repair_io_failure to do the rest for us.
2279 */
2280 if (failrec->in_validation) {
2281 BUG_ON(failrec->this_mirror != failed_mirror);
2282 failrec->in_validation = 0;
2283 failrec->this_mirror = 0;
2284 }
2285 failrec->failed_mirror = failed_mirror;
2286 failrec->this_mirror++;
2287 if (failrec->this_mirror == failed_mirror)
2288 failrec->this_mirror++;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002289 }
2290
Miao Xiefacc8a222013-07-25 19:22:34 +08002291 if (failrec->this_mirror > num_copies) {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002292 btrfs_debug(fs_info,
2293 "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2294 num_copies, failrec->this_mirror, failed_mirror);
Liu Boc3cfb652017-07-13 15:00:50 -07002295 return false;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002296 }
2297
Liu Boc3cfb652017-07-13 15:00:50 -07002298 return true;
Miao Xie2fe63032014-09-12 18:43:59 +08002299}
2300
2301
2302struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2303 struct io_failure_record *failrec,
2304 struct page *page, int pg_offset, int icsum,
Miao Xie8b110e32014-09-12 18:44:03 +08002305 bio_end_io_t *endio_func, void *data)
Miao Xie2fe63032014-09-12 18:43:59 +08002306{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002307 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002308 struct bio *bio;
2309 struct btrfs_io_bio *btrfs_failed_bio;
2310 struct btrfs_io_bio *btrfs_bio;
2311
David Sterbac5e4c3d2017-06-12 17:29:41 +02002312 bio = btrfs_io_bio_alloc(1);
Miao Xie2fe63032014-09-12 18:43:59 +08002313 bio->bi_end_io = endio_func;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002314 bio->bi_iter.bi_sector = failrec->logical >> 9;
Christoph Hellwig74d46992017-08-23 19:10:32 +02002315 bio_set_dev(bio, fs_info->fs_devices->latest_bdev);
Kent Overstreet4f024f32013-10-11 15:44:27 -07002316 bio->bi_iter.bi_size = 0;
Miao Xie8b110e32014-09-12 18:44:03 +08002317 bio->bi_private = data;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002318
Miao Xiefacc8a222013-07-25 19:22:34 +08002319 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2320 if (btrfs_failed_bio->csum) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002321 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2322
2323 btrfs_bio = btrfs_io_bio(bio);
2324 btrfs_bio->csum = btrfs_bio->csum_inline;
Miao Xie2fe63032014-09-12 18:43:59 +08002325 icsum *= csum_size;
2326 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
Miao Xiefacc8a222013-07-25 19:22:34 +08002327 csum_size);
2328 }
2329
Miao Xie2fe63032014-09-12 18:43:59 +08002330 bio_add_page(bio, page, failrec->len, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002331
Miao Xie2fe63032014-09-12 18:43:59 +08002332 return bio;
2333}
2334
2335/*
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002336 * This is a generic handler for readpage errors. If other copies exist, read
2337 * those and write back good data to the failed position. Does not investigate
2338 * in remapping the failed extent elsewhere, hoping the device will be smart
2339 * enough to do this as needed
Miao Xie2fe63032014-09-12 18:43:59 +08002340 */
Miao Xie2fe63032014-09-12 18:43:59 +08002341static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2342 struct page *page, u64 start, u64 end,
2343 int failed_mirror)
2344{
2345 struct io_failure_record *failrec;
2346 struct inode *inode = page->mapping->host;
2347 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Josef Bacik7870d082017-05-05 11:57:15 -04002348 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
Miao Xie2fe63032014-09-12 18:43:59 +08002349 struct bio *bio;
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002350 int read_mode = 0;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002351 blk_status_t status;
Miao Xie2fe63032014-09-12 18:43:59 +08002352 int ret;
Christoph Hellwig8a2ee442019-02-15 19:13:07 +08002353 unsigned failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT;
Miao Xie2fe63032014-09-12 18:43:59 +08002354
Mike Christie1f7ad752016-06-05 14:31:51 -05002355 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
Miao Xie2fe63032014-09-12 18:43:59 +08002356
2357 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2358 if (ret)
2359 return ret;
2360
Ming Leia0b60d72017-12-18 20:22:11 +08002361 if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
Liu Boc3cfb652017-07-13 15:00:50 -07002362 failed_mirror)) {
Josef Bacik7870d082017-05-05 11:57:15 -04002363 free_io_failure(failure_tree, tree, failrec);
Miao Xie2fe63032014-09-12 18:43:59 +08002364 return -EIO;
2365 }
2366
Ming Leia0b60d72017-12-18 20:22:11 +08002367 if (failed_bio_pages > 1)
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002368 read_mode |= REQ_FAILFAST_DEV;
Miao Xie2fe63032014-09-12 18:43:59 +08002369
2370 phy_offset >>= inode->i_sb->s_blocksize_bits;
2371 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2372 start - page_offset(page),
Miao Xie8b110e32014-09-12 18:44:03 +08002373 (int)phy_offset, failed_bio->bi_end_io,
2374 NULL);
David Sterbaebcc3262018-06-29 10:56:53 +02002375 bio->bi_opf = REQ_OP_READ | read_mode;
Miao Xie2fe63032014-09-12 18:43:59 +08002376
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002377 btrfs_debug(btrfs_sb(inode->i_sb),
2378 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2379 read_mode, failrec->this_mirror, failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002380
Linus Torvalds8c27cb32017-07-05 16:41:23 -07002381 status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002382 failrec->bio_flags, 0);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002383 if (status) {
Josef Bacik7870d082017-05-05 11:57:15 -04002384 free_io_failure(failure_tree, tree, failrec);
Miao Xie6c387ab2014-09-12 18:43:57 +08002385 bio_put(bio);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002386 ret = blk_status_to_errno(status);
Miao Xie6c387ab2014-09-12 18:43:57 +08002387 }
2388
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002389 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002390}
2391
Chris Masond1310b22008-01-24 16:13:08 -05002392/* lots and lots of room for performance fixes in the end_bio funcs */
2393
David Sterbab5227c02015-12-03 13:08:59 +01002394void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
Jeff Mahoney87826df2012-02-15 16:23:57 +01002395{
2396 int uptodate = (err == 0);
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002397 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002398
Nikolay Borisovc6297322018-11-08 10:18:08 +02002399 btrfs_writepage_endio_finish_ordered(page, start, end, uptodate);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002400
Jeff Mahoney87826df2012-02-15 16:23:57 +01002401 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002402 ClearPageUptodate(page);
2403 SetPageError(page);
Colin Ian Kingbff5baf2017-05-09 18:14:01 +01002404 ret = err < 0 ? err : -EIO;
Liu Bo5dca6ee2014-05-12 12:47:36 +08002405 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002406 }
Jeff Mahoney87826df2012-02-15 16:23:57 +01002407}
2408
Chris Masond1310b22008-01-24 16:13:08 -05002409/*
2410 * after a writepage IO is done, we need to:
2411 * clear the uptodate bits on error
2412 * clear the writeback bits in the extent tree for this IO
2413 * end_page_writeback if the page has no more pending IO
2414 *
2415 * Scheduling is not allowed, so the extent state tree is expected
2416 * to have one and only one object corresponding to this IO.
2417 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002418static void end_bio_extent_writepage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002419{
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002420 int error = blk_status_to_errno(bio->bi_status);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002421 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002422 u64 start;
2423 u64 end;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002424 int i;
Ming Lei6dc4f102019-02-15 19:13:19 +08002425 struct bvec_iter_all iter_all;
Chris Masond1310b22008-01-24 16:13:08 -05002426
David Sterbac09abff2017-07-13 18:10:07 +02002427 ASSERT(!bio_flagged(bio, BIO_CLONED));
Ming Lei6dc4f102019-02-15 19:13:19 +08002428 bio_for_each_segment_all(bvec, bio, i, iter_all) {
Chris Masond1310b22008-01-24 16:13:08 -05002429 struct page *page = bvec->bv_page;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002430 struct inode *inode = page->mapping->host;
2431 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
David Woodhouse902b22f2008-08-20 08:51:49 -04002432
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002433 /* We always issue full-page reads, but if some block
2434 * in a page fails to read, blk_update_request() will
2435 * advance bv_offset and adjust bv_len to compensate.
2436 * Print a warning for nonzero offsets, and an error
2437 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002438 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2439 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002440 btrfs_err(fs_info,
Frank Holtonefe120a2013-12-20 11:37:06 -05002441 "partial page write in btrfs with offset %u and length %u",
2442 bvec->bv_offset, bvec->bv_len);
2443 else
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002444 btrfs_info(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04002445 "incomplete page write in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002446 bvec->bv_offset, bvec->bv_len);
2447 }
Chris Masond1310b22008-01-24 16:13:08 -05002448
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002449 start = page_offset(page);
2450 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002451
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002452 end_extent_writepage(page, error, start, end);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002453 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002454 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002455
Chris Masond1310b22008-01-24 16:13:08 -05002456 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002457}
2458
Miao Xie883d0de2013-07-25 19:22:35 +08002459static void
2460endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2461 int uptodate)
2462{
2463 struct extent_state *cached = NULL;
2464 u64 end = start + len - 1;
2465
2466 if (uptodate && tree->track_uptodate)
2467 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
David Sterbad810a4b2017-12-07 18:52:54 +01002468 unlock_extent_cached_atomic(tree, start, end, &cached);
Miao Xie883d0de2013-07-25 19:22:35 +08002469}
2470
Chris Masond1310b22008-01-24 16:13:08 -05002471/*
2472 * after a readpage IO is done, we need to:
2473 * clear the uptodate bits on error
2474 * set the uptodate bits if things worked
2475 * set the page up to date if all extents in the tree are uptodate
2476 * clear the lock bit in the extent tree
2477 * unlock the page if there are no other extents locked for it
2478 *
2479 * Scheduling is not allowed, so the extent state tree is expected
2480 * to have one and only one object corresponding to this IO.
2481 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002482static void end_bio_extent_readpage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002483{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002484 struct bio_vec *bvec;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002485 int uptodate = !bio->bi_status;
Miao Xiefacc8a222013-07-25 19:22:34 +08002486 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
Josef Bacik7870d082017-05-05 11:57:15 -04002487 struct extent_io_tree *tree, *failure_tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002488 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002489 u64 start;
2490 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002491 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002492 u64 extent_start = 0;
2493 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002494 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002495 int ret;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002496 int i;
Ming Lei6dc4f102019-02-15 19:13:19 +08002497 struct bvec_iter_all iter_all;
Chris Masond1310b22008-01-24 16:13:08 -05002498
David Sterbac09abff2017-07-13 18:10:07 +02002499 ASSERT(!bio_flagged(bio, BIO_CLONED));
Ming Lei6dc4f102019-02-15 19:13:19 +08002500 bio_for_each_segment_all(bvec, bio, i, iter_all) {
Chris Masond1310b22008-01-24 16:13:08 -05002501 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002502 struct inode *inode = page->mapping->host;
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002503 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002504 bool data_inode = btrfs_ino(BTRFS_I(inode))
2505 != BTRFS_BTREE_INODE_OBJECTID;
Arne Jansen507903b2011-04-06 10:02:20 +00002506
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002507 btrfs_debug(fs_info,
2508 "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002509 (u64)bio->bi_iter.bi_sector, bio->bi_status,
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002510 io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002511 tree = &BTRFS_I(inode)->io_tree;
Josef Bacik7870d082017-05-05 11:57:15 -04002512 failure_tree = &BTRFS_I(inode)->io_failure_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002513
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002514 /* We always issue full-page reads, but if some block
2515 * in a page fails to read, blk_update_request() will
2516 * advance bv_offset and adjust bv_len to compensate.
2517 * Print a warning for nonzero offsets, and an error
2518 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002519 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2520 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002521 btrfs_err(fs_info,
2522 "partial page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002523 bvec->bv_offset, bvec->bv_len);
2524 else
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002525 btrfs_info(fs_info,
2526 "incomplete page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002527 bvec->bv_offset, bvec->bv_len);
2528 }
Chris Masond1310b22008-01-24 16:13:08 -05002529
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002530 start = page_offset(page);
2531 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002532 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002533
Chris Mason9be33952013-05-17 18:30:14 -04002534 mirror = io_bio->mirror_num;
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002535 if (likely(uptodate)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002536 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2537 page, start, end,
2538 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002539 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002540 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002541 else
Josef Bacik7870d082017-05-05 11:57:15 -04002542 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2543 failure_tree, tree, start,
2544 page,
2545 btrfs_ino(BTRFS_I(inode)), 0);
Chris Masond1310b22008-01-24 16:13:08 -05002546 }
Josef Bacikea466792012-03-26 21:57:36 -04002547
Miao Xief2a09da2013-07-25 19:22:33 +08002548 if (likely(uptodate))
2549 goto readpage_ok;
2550
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002551 if (data_inode) {
Liu Bo9d0d1c82017-03-24 15:04:50 -07002552
2553 /*
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002554 * The generic bio_readpage_error handles errors the
2555 * following way: If possible, new read requests are
2556 * created and submitted and will end up in
2557 * end_bio_extent_readpage as well (if we're lucky,
2558 * not in the !uptodate case). In that case it returns
2559 * 0 and we just go on with the next page in our bio.
2560 * If it can't handle the error it will return -EIO and
2561 * we remain responsible for that page.
Liu Bo9d0d1c82017-03-24 15:04:50 -07002562 */
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002563 ret = bio_readpage_error(bio, offset, page, start, end,
2564 mirror);
2565 if (ret == 0) {
2566 uptodate = !bio->bi_status;
2567 offset += len;
2568 continue;
2569 }
2570 } else {
2571 struct extent_buffer *eb;
2572
2573 eb = (struct extent_buffer *)page->private;
2574 set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
2575 eb->read_mirror = mirror;
2576 atomic_dec(&eb->io_pages);
2577 if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD,
2578 &eb->bflags))
2579 btree_readahead_hook(eb, -EIO);
2580
2581 ret = -EIO;
Chris Mason7e383262008-04-09 16:28:12 -04002582 }
Miao Xief2a09da2013-07-25 19:22:33 +08002583readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002584 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002585 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002586 pgoff_t end_index = i_size >> PAGE_SHIFT;
Liu Boa583c022014-08-19 23:32:22 +08002587 unsigned off;
Josef Bacika71754f2013-06-17 17:14:39 -04002588
2589 /* Zero out the end if this page straddles i_size */
Johannes Thumshirn70730172018-12-05 15:23:03 +01002590 off = offset_in_page(i_size);
Liu Boa583c022014-08-19 23:32:22 +08002591 if (page->index == end_index && off)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002592 zero_user_segment(page, off, PAGE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002593 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002594 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002595 ClearPageUptodate(page);
2596 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002597 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002598 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002599 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002600
2601 if (unlikely(!uptodate)) {
2602 if (extent_len) {
2603 endio_readpage_release_extent(tree,
2604 extent_start,
2605 extent_len, 1);
2606 extent_start = 0;
2607 extent_len = 0;
2608 }
2609 endio_readpage_release_extent(tree, start,
2610 end - start + 1, 0);
2611 } else if (!extent_len) {
2612 extent_start = start;
2613 extent_len = end + 1 - start;
2614 } else if (extent_start + extent_len == start) {
2615 extent_len += end + 1 - start;
2616 } else {
2617 endio_readpage_release_extent(tree, extent_start,
2618 extent_len, uptodate);
2619 extent_start = start;
2620 extent_len = end + 1 - start;
2621 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002622 }
Chris Masond1310b22008-01-24 16:13:08 -05002623
Miao Xie883d0de2013-07-25 19:22:35 +08002624 if (extent_len)
2625 endio_readpage_release_extent(tree, extent_start, extent_len,
2626 uptodate);
David Sterbab3a0dd52018-11-22 17:16:49 +01002627 btrfs_io_bio_free_csum(io_bio);
Chris Masond1310b22008-01-24 16:13:08 -05002628 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002629}
2630
Chris Mason9be33952013-05-17 18:30:14 -04002631/*
David Sterba184f9992017-06-12 17:29:39 +02002632 * Initialize the members up to but not including 'bio'. Use after allocating a
2633 * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2634 * 'bio' because use of __GFP_ZERO is not supported.
Chris Mason9be33952013-05-17 18:30:14 -04002635 */
David Sterba184f9992017-06-12 17:29:39 +02002636static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
Chris Masond1310b22008-01-24 16:13:08 -05002637{
David Sterba184f9992017-06-12 17:29:39 +02002638 memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2639}
2640
2641/*
David Sterba6e707bc2017-06-02 17:26:26 +02002642 * The following helpers allocate a bio. As it's backed by a bioset, it'll
2643 * never fail. We're returning a bio right now but you can call btrfs_io_bio
2644 * for the appropriate container_of magic
Chris Masond1310b22008-01-24 16:13:08 -05002645 */
David Sterbac821e7f32017-06-02 18:35:36 +02002646struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte)
Chris Masond1310b22008-01-24 16:13:08 -05002647{
2648 struct bio *bio;
2649
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002650 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
Christoph Hellwig74d46992017-08-23 19:10:32 +02002651 bio_set_dev(bio, bdev);
David Sterbac821e7f32017-06-02 18:35:36 +02002652 bio->bi_iter.bi_sector = first_byte >> 9;
David Sterba184f9992017-06-12 17:29:39 +02002653 btrfs_io_bio_init(btrfs_io_bio(bio));
Chris Masond1310b22008-01-24 16:13:08 -05002654 return bio;
2655}
2656
David Sterba8b6c1d52017-06-02 17:48:13 +02002657struct bio *btrfs_bio_clone(struct bio *bio)
Chris Mason9be33952013-05-17 18:30:14 -04002658{
Miao Xie23ea8e52014-09-12 18:43:54 +08002659 struct btrfs_io_bio *btrfs_bio;
2660 struct bio *new;
Chris Mason9be33952013-05-17 18:30:14 -04002661
David Sterba6e707bc2017-06-02 17:26:26 +02002662 /* Bio allocation backed by a bioset does not fail */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002663 new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
David Sterba6e707bc2017-06-02 17:26:26 +02002664 btrfs_bio = btrfs_io_bio(new);
David Sterba184f9992017-06-12 17:29:39 +02002665 btrfs_io_bio_init(btrfs_bio);
David Sterba6e707bc2017-06-02 17:26:26 +02002666 btrfs_bio->iter = bio->bi_iter;
Miao Xie23ea8e52014-09-12 18:43:54 +08002667 return new;
2668}
Chris Mason9be33952013-05-17 18:30:14 -04002669
David Sterbac5e4c3d2017-06-12 17:29:41 +02002670struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
Chris Mason9be33952013-05-17 18:30:14 -04002671{
Miao Xiefacc8a222013-07-25 19:22:34 +08002672 struct bio *bio;
2673
David Sterba6e707bc2017-06-02 17:26:26 +02002674 /* Bio allocation backed by a bioset does not fail */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002675 bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
David Sterba184f9992017-06-12 17:29:39 +02002676 btrfs_io_bio_init(btrfs_io_bio(bio));
Miao Xiefacc8a222013-07-25 19:22:34 +08002677 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002678}
2679
Liu Boe4770942017-05-16 10:57:14 -07002680struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
Liu Bo2f8e9142017-05-15 17:43:31 -07002681{
2682 struct bio *bio;
2683 struct btrfs_io_bio *btrfs_bio;
2684
2685 /* this will never fail when it's backed by a bioset */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002686 bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
Liu Bo2f8e9142017-05-15 17:43:31 -07002687 ASSERT(bio);
2688
2689 btrfs_bio = btrfs_io_bio(bio);
David Sterba184f9992017-06-12 17:29:39 +02002690 btrfs_io_bio_init(btrfs_bio);
Liu Bo2f8e9142017-05-15 17:43:31 -07002691
2692 bio_trim(bio, offset >> 9, size >> 9);
Liu Bo17347ce2017-05-15 15:33:27 -07002693 btrfs_bio->iter = bio->bi_iter;
Liu Bo2f8e9142017-05-15 17:43:31 -07002694 return bio;
2695}
Chris Mason9be33952013-05-17 18:30:14 -04002696
Mike Christie1f7ad752016-06-05 14:31:51 -05002697static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
2698 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002699{
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002700 blk_status_t ret = 0;
Ming Leic45a8f22017-12-18 20:22:05 +08002701 struct bio_vec *bvec = bio_last_bvec_all(bio);
Ming Leic3a7ce72019-02-15 19:13:16 +08002702 struct bio_vec bv;
Chris Mason70dec802008-01-29 09:59:12 -05002703 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002704 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002705
Ming Leic3a7ce72019-02-15 19:13:16 +08002706 mp_bvec_last_segment(bvec, &bv);
2707 start = page_offset(bv.bv_page) + bv.bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002708
David Woodhouse902b22f2008-08-20 08:51:49 -04002709 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002710
David Sterba20c98012017-02-17 15:59:35 +01002711 if (tree->ops)
Josef Bacikc6100a42017-05-05 11:57:13 -04002712 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002713 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002714 else
Mike Christie4e49ea42016-06-05 14:31:41 -05002715 btrfsic_submit_bio(bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002716
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002717 return blk_status_to_errno(ret);
Chris Masond1310b22008-01-24 16:13:08 -05002718}
2719
David Sterba4b81ba42017-06-06 19:14:26 +02002720/*
2721 * @opf: bio REQ_OP_* and REQ_* flags as one value
David Sterbab8b3d622017-06-12 19:50:41 +02002722 * @tree: tree so we can call our merge_bio hook
2723 * @wbc: optional writeback control for io accounting
2724 * @page: page to add to the bio
2725 * @pg_offset: offset of the new bio or to check whether we are adding
2726 * a contiguous page to the previous one
2727 * @size: portion of page that we want to write
2728 * @offset: starting offset in the page
2729 * @bdev: attach newly created bios to this bdev
David Sterba5c2b1fd2017-06-06 19:22:55 +02002730 * @bio_ret: must be valid pointer, newly allocated bio will be stored there
David Sterbab8b3d622017-06-12 19:50:41 +02002731 * @end_io_func: end_io callback for new bio
2732 * @mirror_num: desired mirror to read/write
2733 * @prev_bio_flags: flags of previous bio to see if we can merge the current one
2734 * @bio_flags: flags of the current bio to see if we can merge them
David Sterba4b81ba42017-06-06 19:14:26 +02002735 */
2736static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
Chris Masonda2f0f72015-07-02 13:57:22 -07002737 struct writeback_control *wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02002738 struct page *page, u64 offset,
David Sterba6c5a4e22017-10-04 17:10:34 +02002739 size_t size, unsigned long pg_offset,
Chris Masond1310b22008-01-24 16:13:08 -05002740 struct block_device *bdev,
2741 struct bio **bio_ret,
Chris Masonf1885912008-04-09 16:28:12 -04002742 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002743 int mirror_num,
2744 unsigned long prev_bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002745 unsigned long bio_flags,
2746 bool force_bio_submit)
Chris Masond1310b22008-01-24 16:13:08 -05002747{
2748 int ret = 0;
2749 struct bio *bio;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002750 size_t page_size = min_t(size_t, size, PAGE_SIZE);
David Sterba6273b7f2017-10-04 17:30:11 +02002751 sector_t sector = offset >> 9;
Chris Masond1310b22008-01-24 16:13:08 -05002752
David Sterba5c2b1fd2017-06-06 19:22:55 +02002753 ASSERT(bio_ret);
2754
2755 if (*bio_ret) {
David Sterba0c8508a2017-06-12 20:00:43 +02002756 bool contig;
2757 bool can_merge = true;
2758
Chris Masond1310b22008-01-24 16:13:08 -05002759 bio = *bio_ret;
David Sterba0c8508a2017-06-12 20:00:43 +02002760 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
Kent Overstreet4f024f32013-10-11 15:44:27 -07002761 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002762 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002763 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002764
Nikolay Borisovda12fe52018-11-27 20:57:58 +02002765 ASSERT(tree->ops);
2766 if (btrfs_bio_fits_in_stripe(page, page_size, bio, bio_flags))
David Sterba0c8508a2017-06-12 20:00:43 +02002767 can_merge = false;
2768
2769 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
Filipe Manana005efed2015-09-14 09:09:31 +01002770 force_bio_submit ||
David Sterba6c5a4e22017-10-04 17:10:34 +02002771 bio_add_page(bio, page, page_size, pg_offset) < page_size) {
Mike Christie1f7ad752016-06-05 14:31:51 -05002772 ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
Naohiro Aota289454a2015-01-06 01:01:03 +09002773 if (ret < 0) {
2774 *bio_ret = NULL;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002775 return ret;
Naohiro Aota289454a2015-01-06 01:01:03 +09002776 }
Chris Masond1310b22008-01-24 16:13:08 -05002777 bio = NULL;
2778 } else {
Chris Masonda2f0f72015-07-02 13:57:22 -07002779 if (wbc)
2780 wbc_account_io(wbc, page, page_size);
Chris Masond1310b22008-01-24 16:13:08 -05002781 return 0;
2782 }
2783 }
Chris Masonc8b97812008-10-29 14:49:59 -04002784
David Sterba6273b7f2017-10-04 17:30:11 +02002785 bio = btrfs_bio_alloc(bdev, offset);
David Sterba6c5a4e22017-10-04 17:10:34 +02002786 bio_add_page(bio, page, page_size, pg_offset);
Chris Masond1310b22008-01-24 16:13:08 -05002787 bio->bi_end_io = end_io_func;
2788 bio->bi_private = tree;
Jens Axboee6959b92017-06-27 11:51:28 -06002789 bio->bi_write_hint = page->mapping->host->i_write_hint;
David Sterba4b81ba42017-06-06 19:14:26 +02002790 bio->bi_opf = opf;
Chris Masonda2f0f72015-07-02 13:57:22 -07002791 if (wbc) {
2792 wbc_init_bio(wbc, bio);
2793 wbc_account_io(wbc, page, page_size);
2794 }
Chris Mason70dec802008-01-29 09:59:12 -05002795
David Sterba5c2b1fd2017-06-06 19:22:55 +02002796 *bio_ret = bio;
Chris Masond1310b22008-01-24 16:13:08 -05002797
2798 return ret;
2799}
2800
Eric Sandeen48a3b632013-04-25 20:41:01 +00002801static void attach_extent_buffer_page(struct extent_buffer *eb,
2802 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002803{
2804 if (!PagePrivate(page)) {
2805 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002806 get_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002807 set_page_private(page, (unsigned long)eb);
2808 } else {
2809 WARN_ON(page->private != (unsigned long)eb);
2810 }
2811}
2812
Chris Masond1310b22008-01-24 16:13:08 -05002813void set_page_extent_mapped(struct page *page)
2814{
2815 if (!PagePrivate(page)) {
2816 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002817 get_page(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002818 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002819 }
2820}
2821
Miao Xie125bac012013-07-25 19:22:37 +08002822static struct extent_map *
2823__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2824 u64 start, u64 len, get_extent_t *get_extent,
2825 struct extent_map **em_cached)
2826{
2827 struct extent_map *em;
2828
2829 if (em_cached && *em_cached) {
2830 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00002831 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08002832 start < extent_map_end(em)) {
Elena Reshetova490b54d2017-03-03 10:55:12 +02002833 refcount_inc(&em->refs);
Miao Xie125bac012013-07-25 19:22:37 +08002834 return em;
2835 }
2836
2837 free_extent_map(em);
2838 *em_cached = NULL;
2839 }
2840
Nikolay Borisovfc4f21b12017-02-20 13:51:06 +02002841 em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
Miao Xie125bac012013-07-25 19:22:37 +08002842 if (em_cached && !IS_ERR_OR_NULL(em)) {
2843 BUG_ON(*em_cached);
Elena Reshetova490b54d2017-03-03 10:55:12 +02002844 refcount_inc(&em->refs);
Miao Xie125bac012013-07-25 19:22:37 +08002845 *em_cached = em;
2846 }
2847 return em;
2848}
Chris Masond1310b22008-01-24 16:13:08 -05002849/*
2850 * basic readpage implementation. Locked extent state structs are inserted
2851 * into the tree that are removed when the IO is done (by the end_io
2852 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002853 * XXX JDM: This needs looking at to ensure proper page locking
Liu Bobaf863b2016-07-11 10:39:07 -07002854 * return 0 on success, otherwise return error
Chris Masond1310b22008-01-24 16:13:08 -05002855 */
Miao Xie99740902013-07-25 19:22:36 +08002856static int __do_readpage(struct extent_io_tree *tree,
2857 struct page *page,
2858 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002859 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002860 struct bio **bio, int mirror_num,
David Sterbaf1c77c52017-06-06 19:03:49 +02002861 unsigned long *bio_flags, unsigned int read_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002862 u64 *prev_em_start)
Chris Masond1310b22008-01-24 16:13:08 -05002863{
2864 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002865 u64 start = page_offset(page);
David Sterba8eec8292017-06-06 19:50:13 +02002866 const u64 end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002867 u64 cur = start;
2868 u64 extent_offset;
2869 u64 last_byte = i_size_read(inode);
2870 u64 block_start;
2871 u64 cur_end;
Chris Masond1310b22008-01-24 16:13:08 -05002872 struct extent_map *em;
2873 struct block_device *bdev;
Liu Bobaf863b2016-07-11 10:39:07 -07002874 int ret = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002875 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02002876 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002877 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002878 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002879 size_t blocksize = inode->i_sb->s_blocksize;
Filipe Manana7f042a82016-01-27 19:17:20 +00002880 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002881
2882 set_page_extent_mapped(page);
2883
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002884 if (!PageUptodate(page)) {
2885 if (cleancache_get_page(page) == 0) {
2886 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08002887 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002888 goto out;
2889 }
2890 }
2891
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002892 if (page->index == last_byte >> PAGE_SHIFT) {
Chris Masonc8b97812008-10-29 14:49:59 -04002893 char *userpage;
Johannes Thumshirn70730172018-12-05 15:23:03 +01002894 size_t zero_offset = offset_in_page(last_byte);
Chris Masonc8b97812008-10-29 14:49:59 -04002895
2896 if (zero_offset) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002897 iosize = PAGE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002898 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002899 memset(userpage + zero_offset, 0, iosize);
2900 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002901 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002902 }
2903 }
Chris Masond1310b22008-01-24 16:13:08 -05002904 while (cur <= end) {
Filipe Manana005efed2015-09-14 09:09:31 +01002905 bool force_bio_submit = false;
David Sterba6273b7f2017-10-04 17:30:11 +02002906 u64 offset;
Josef Bacikc8f2f242013-02-11 11:33:00 -05002907
Chris Masond1310b22008-01-24 16:13:08 -05002908 if (cur >= last_byte) {
2909 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002910 struct extent_state *cached = NULL;
2911
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002912 iosize = PAGE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002913 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002914 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002915 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002916 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002917 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002918 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00002919 unlock_extent_cached(tree, cur,
David Sterbae43bbe52017-12-12 21:43:52 +01002920 cur + iosize - 1, &cached);
Chris Masond1310b22008-01-24 16:13:08 -05002921 break;
2922 }
Miao Xie125bac012013-07-25 19:22:37 +08002923 em = __get_extent_map(inode, page, pg_offset, cur,
2924 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02002925 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002926 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00002927 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002928 break;
2929 }
Chris Masond1310b22008-01-24 16:13:08 -05002930 extent_offset = cur - em->start;
2931 BUG_ON(extent_map_end(em) <= cur);
2932 BUG_ON(end < cur);
2933
Li Zefan261507a02010-12-17 14:21:50 +08002934 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07002935 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002936 extent_set_compress_type(&this_bio_flag,
2937 em->compress_type);
2938 }
Chris Masonc8b97812008-10-29 14:49:59 -04002939
Chris Masond1310b22008-01-24 16:13:08 -05002940 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2941 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00002942 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04002943 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2944 disk_io_size = em->block_len;
David Sterba6273b7f2017-10-04 17:30:11 +02002945 offset = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04002946 } else {
David Sterba6273b7f2017-10-04 17:30:11 +02002947 offset = em->block_start + extent_offset;
Chris Masonc8b97812008-10-29 14:49:59 -04002948 disk_io_size = iosize;
2949 }
Chris Masond1310b22008-01-24 16:13:08 -05002950 bdev = em->bdev;
2951 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002952 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2953 block_start = EXTENT_MAP_HOLE;
Filipe Manana005efed2015-09-14 09:09:31 +01002954
2955 /*
2956 * If we have a file range that points to a compressed extent
2957 * and it's followed by a consecutive file range that points to
2958 * to the same compressed extent (possibly with a different
2959 * offset and/or length, so it either points to the whole extent
2960 * or only part of it), we must make sure we do not submit a
2961 * single bio to populate the pages for the 2 ranges because
2962 * this makes the compressed extent read zero out the pages
2963 * belonging to the 2nd range. Imagine the following scenario:
2964 *
2965 * File layout
2966 * [0 - 8K] [8K - 24K]
2967 * | |
2968 * | |
2969 * points to extent X, points to extent X,
2970 * offset 4K, length of 8K offset 0, length 16K
2971 *
2972 * [extent X, compressed length = 4K uncompressed length = 16K]
2973 *
2974 * If the bio to read the compressed extent covers both ranges,
2975 * it will decompress extent X into the pages belonging to the
2976 * first range and then it will stop, zeroing out the remaining
2977 * pages that belong to the other range that points to extent X.
2978 * So here we make sure we submit 2 bios, one for the first
2979 * range and another one for the third range. Both will target
2980 * the same physical extent from disk, but we can't currently
2981 * make the compressed bio endio callback populate the pages
2982 * for both ranges because each compressed bio is tightly
2983 * coupled with a single extent map, and each range can have
2984 * an extent map with a different offset value relative to the
2985 * uncompressed data of our extent and different lengths. This
2986 * is a corner case so we prioritize correctness over
2987 * non-optimal behavior (submitting 2 bios for the same extent).
2988 */
2989 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
2990 prev_em_start && *prev_em_start != (u64)-1 &&
2991 *prev_em_start != em->orig_start)
2992 force_bio_submit = true;
2993
2994 if (prev_em_start)
2995 *prev_em_start = em->orig_start;
2996
Chris Masond1310b22008-01-24 16:13:08 -05002997 free_extent_map(em);
2998 em = NULL;
2999
3000 /* we've found a hole, just zero and go on */
3001 if (block_start == EXTENT_MAP_HOLE) {
3002 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003003 struct extent_state *cached = NULL;
3004
Cong Wang7ac687d2011-11-25 23:14:28 +08003005 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003006 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003007 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003008 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003009
3010 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003011 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00003012 unlock_extent_cached(tree, cur,
David Sterbae43bbe52017-12-12 21:43:52 +01003013 cur + iosize - 1, &cached);
Chris Masond1310b22008-01-24 16:13:08 -05003014 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003015 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003016 continue;
3017 }
3018 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04003019 if (test_range_bit(tree, cur, cur_end,
3020 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04003021 check_page_uptodate(tree, page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003022 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003023 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003024 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003025 continue;
3026 }
Chris Mason70dec802008-01-29 09:59:12 -05003027 /* we have an inline extent but it didn't get marked up
3028 * to date. Error out
3029 */
3030 if (block_start == EXTENT_MAP_INLINE) {
3031 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003032 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05003033 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003034 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05003035 continue;
3036 }
Chris Masond1310b22008-01-24 16:13:08 -05003037
David Sterba4b81ba42017-06-06 19:14:26 +02003038 ret = submit_extent_page(REQ_OP_READ | read_flags, tree, NULL,
David Sterba6273b7f2017-10-04 17:30:11 +02003039 page, offset, disk_io_size,
3040 pg_offset, bdev, bio,
Chris Masonc8b97812008-10-29 14:49:59 -04003041 end_bio_extent_readpage, mirror_num,
3042 *bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01003043 this_bio_flag,
3044 force_bio_submit);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003045 if (!ret) {
3046 nr++;
3047 *bio_flags = this_bio_flag;
3048 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003049 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003050 unlock_extent(tree, cur, cur + iosize - 1);
Liu Bobaf863b2016-07-11 10:39:07 -07003051 goto out;
Josef Bacikedd33c92012-10-05 16:40:32 -04003052 }
Chris Masond1310b22008-01-24 16:13:08 -05003053 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003054 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003055 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003056out:
Chris Masond1310b22008-01-24 16:13:08 -05003057 if (!nr) {
3058 if (!PageError(page))
3059 SetPageUptodate(page);
3060 unlock_page(page);
3061 }
Liu Bobaf863b2016-07-11 10:39:07 -07003062 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05003063}
3064
Miao Xie99740902013-07-25 19:22:36 +08003065static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3066 struct page *pages[], int nr_pages,
3067 u64 start, u64 end,
Miao Xie125bac012013-07-25 19:22:37 +08003068 struct extent_map **em_cached,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003069 struct bio **bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003070 unsigned long *bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003071 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003072{
3073 struct inode *inode;
3074 struct btrfs_ordered_extent *ordered;
3075 int index;
3076
3077 inode = pages[0]->mapping->host;
3078 while (1) {
3079 lock_extent(tree, start, end);
Nikolay Borisova776c6f2017-02-20 13:50:49 +02003080 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
Miao Xie99740902013-07-25 19:22:36 +08003081 end - start + 1);
3082 if (!ordered)
3083 break;
3084 unlock_extent(tree, start, end);
3085 btrfs_start_ordered_extent(inode, ordered, 1);
3086 btrfs_put_ordered_extent(ordered);
3087 }
3088
3089 for (index = 0; index < nr_pages; index++) {
David Sterba4ef77692017-06-23 04:09:57 +02003090 __do_readpage(tree, pages[index], btrfs_get_extent, em_cached,
Jens Axboe5e9d3982018-08-17 15:45:39 -07003091 bio, 0, bio_flags, REQ_RAHEAD, prev_em_start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003092 put_page(pages[index]);
Miao Xie99740902013-07-25 19:22:36 +08003093 }
3094}
3095
3096static void __extent_readpages(struct extent_io_tree *tree,
3097 struct page *pages[],
David Sterbae4d17ef2017-06-23 04:09:57 +02003098 int nr_pages,
Miao Xie125bac012013-07-25 19:22:37 +08003099 struct extent_map **em_cached,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003100 struct bio **bio, unsigned long *bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003101 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003102{
Stefan Behrens35a36212013-08-14 18:12:25 +02003103 u64 start = 0;
Miao Xie99740902013-07-25 19:22:36 +08003104 u64 end = 0;
3105 u64 page_start;
3106 int index;
Stefan Behrens35a36212013-08-14 18:12:25 +02003107 int first_index = 0;
Miao Xie99740902013-07-25 19:22:36 +08003108
3109 for (index = 0; index < nr_pages; index++) {
3110 page_start = page_offset(pages[index]);
3111 if (!end) {
3112 start = page_start;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003113 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003114 first_index = index;
3115 } else if (end + 1 == page_start) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003116 end += PAGE_SIZE;
Miao Xie99740902013-07-25 19:22:36 +08003117 } else {
3118 __do_contiguous_readpages(tree, &pages[first_index],
3119 index - first_index, start,
David Sterba4ef77692017-06-23 04:09:57 +02003120 end, em_cached,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003121 bio, bio_flags,
Mike Christie1f7ad752016-06-05 14:31:51 -05003122 prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003123 start = page_start;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003124 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003125 first_index = index;
3126 }
3127 }
3128
3129 if (end)
3130 __do_contiguous_readpages(tree, &pages[first_index],
3131 index - first_index, start,
David Sterba4ef77692017-06-23 04:09:57 +02003132 end, em_cached, bio,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003133 bio_flags, prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003134}
3135
3136static int __extent_read_full_page(struct extent_io_tree *tree,
3137 struct page *page,
3138 get_extent_t *get_extent,
3139 struct bio **bio, int mirror_num,
David Sterbaf1c77c52017-06-06 19:03:49 +02003140 unsigned long *bio_flags,
3141 unsigned int read_flags)
Miao Xie99740902013-07-25 19:22:36 +08003142{
3143 struct inode *inode = page->mapping->host;
3144 struct btrfs_ordered_extent *ordered;
3145 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003146 u64 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003147 int ret;
3148
3149 while (1) {
3150 lock_extent(tree, start, end);
Nikolay Borisova776c6f2017-02-20 13:50:49 +02003151 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003152 PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08003153 if (!ordered)
3154 break;
3155 unlock_extent(tree, start, end);
3156 btrfs_start_ordered_extent(inode, ordered, 1);
3157 btrfs_put_ordered_extent(ordered);
3158 }
3159
Miao Xie125bac012013-07-25 19:22:37 +08003160 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003161 bio_flags, read_flags, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003162 return ret;
3163}
3164
Chris Masond1310b22008-01-24 16:13:08 -05003165int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003166 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003167{
3168 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003169 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003170 int ret;
3171
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003172 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003173 &bio_flags, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003174 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05003175 ret = submit_one_bio(bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003176 return ret;
3177}
Chris Masond1310b22008-01-24 16:13:08 -05003178
David Sterba3d4b9492017-02-10 19:33:41 +01003179static void update_nr_written(struct writeback_control *wbc,
Liu Boa91326672016-03-07 16:56:21 -08003180 unsigned long nr_written)
Chris Mason11c83492009-04-20 15:50:09 -04003181{
3182 wbc->nr_to_write -= nr_written;
Chris Mason11c83492009-04-20 15:50:09 -04003183}
3184
Chris Masond1310b22008-01-24 16:13:08 -05003185/*
Chris Mason40f76582014-05-21 13:35:51 -07003186 * helper for __extent_writepage, doing all of the delayed allocation setup.
3187 *
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003188 * This returns 1 if btrfs_run_delalloc_range function did all the work required
Chris Mason40f76582014-05-21 13:35:51 -07003189 * to write the page (copy into inline extent). In this case the IO has
3190 * been started and the page is already unlocked.
3191 *
3192 * This returns 0 if all went well (page still locked)
3193 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003194 */
Chris Mason40f76582014-05-21 13:35:51 -07003195static noinline_for_stack int writepage_delalloc(struct inode *inode,
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003196 struct page *page, struct writeback_control *wbc,
3197 u64 delalloc_start, unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003198{
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003199 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003200 u64 page_end = delalloc_start + PAGE_SIZE - 1;
Lu Fengqi3522e902018-11-29 11:33:38 +08003201 bool found;
Chris Mason40f76582014-05-21 13:35:51 -07003202 u64 delalloc_to_write = 0;
3203 u64 delalloc_end = 0;
3204 int ret;
3205 int page_started = 0;
3206
Chris Mason40f76582014-05-21 13:35:51 -07003207
3208 while (delalloc_end < page_end) {
Lu Fengqi3522e902018-11-29 11:33:38 +08003209 found = find_lock_delalloc_range(inode, tree,
Chris Mason40f76582014-05-21 13:35:51 -07003210 page,
3211 &delalloc_start,
Nikolay Borisov917aace2018-10-26 14:43:20 +03003212 &delalloc_end);
Lu Fengqi3522e902018-11-29 11:33:38 +08003213 if (!found) {
Chris Mason40f76582014-05-21 13:35:51 -07003214 delalloc_start = delalloc_end + 1;
3215 continue;
3216 }
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003217 ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
3218 delalloc_end, &page_started, nr_written, wbc);
Chris Mason40f76582014-05-21 13:35:51 -07003219 /* File system has been set read-only */
3220 if (ret) {
3221 SetPageError(page);
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003222 /*
3223 * btrfs_run_delalloc_range should return < 0 for error
3224 * but just in case, we use > 0 here meaning the IO is
3225 * started, so we don't want to return > 0 unless
3226 * things are going well.
Chris Mason40f76582014-05-21 13:35:51 -07003227 */
3228 ret = ret < 0 ? ret : -EIO;
3229 goto done;
3230 }
3231 /*
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003232 * delalloc_end is already one less than the total length, so
3233 * we don't subtract one from PAGE_SIZE
Chris Mason40f76582014-05-21 13:35:51 -07003234 */
3235 delalloc_to_write += (delalloc_end - delalloc_start +
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003236 PAGE_SIZE) >> PAGE_SHIFT;
Chris Mason40f76582014-05-21 13:35:51 -07003237 delalloc_start = delalloc_end + 1;
3238 }
3239 if (wbc->nr_to_write < delalloc_to_write) {
3240 int thresh = 8192;
3241
3242 if (delalloc_to_write < thresh * 2)
3243 thresh = delalloc_to_write;
3244 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3245 thresh);
3246 }
3247
3248 /* did the fill delalloc function already unlock and start
3249 * the IO?
3250 */
3251 if (page_started) {
3252 /*
3253 * we've unlocked the page, so we can't update
3254 * the mapping's writeback index, just update
3255 * nr_to_write.
3256 */
3257 wbc->nr_to_write -= *nr_written;
3258 return 1;
3259 }
3260
3261 ret = 0;
3262
3263done:
3264 return ret;
3265}
3266
3267/*
3268 * helper for __extent_writepage. This calls the writepage start hooks,
3269 * and does the loop to map the page into extents and bios.
3270 *
3271 * We return 1 if the IO is started and the page is unlocked,
3272 * 0 if all went well (page still locked)
3273 * < 0 if there were errors (page still locked)
3274 */
3275static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3276 struct page *page,
3277 struct writeback_control *wbc,
3278 struct extent_page_data *epd,
3279 loff_t i_size,
3280 unsigned long nr_written,
David Sterbaf1c77c52017-06-06 19:03:49 +02003281 unsigned int write_flags, int *nr_ret)
Chris Mason40f76582014-05-21 13:35:51 -07003282{
Chris Masond1310b22008-01-24 16:13:08 -05003283 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003284 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003285 u64 page_end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05003286 u64 end;
3287 u64 cur = start;
3288 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003289 u64 block_start;
3290 u64 iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003291 struct extent_map *em;
3292 struct block_device *bdev;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003293 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003294 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003295 int ret = 0;
3296 int nr = 0;
3297 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003298
Nikolay Borisovd75855b2018-11-01 14:09:47 +02003299 ret = btrfs_writepage_cow_fixup(page, start, page_end);
3300 if (ret) {
3301 /* Fixup worker will requeue */
3302 if (ret == -EBUSY)
3303 wbc->pages_skipped++;
3304 else
3305 redirty_page_for_writepage(wbc, page);
Chris Mason40f76582014-05-21 13:35:51 -07003306
Nikolay Borisovd75855b2018-11-01 14:09:47 +02003307 update_nr_written(wbc, nr_written);
3308 unlock_page(page);
3309 return 1;
Chris Mason247e7432008-07-17 12:53:51 -04003310 }
3311
Chris Mason11c83492009-04-20 15:50:09 -04003312 /*
3313 * we don't want to touch the inode after unlocking the page,
3314 * so we update the mapping writeback index now
3315 */
David Sterba3d4b9492017-02-10 19:33:41 +01003316 update_nr_written(wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003317
Chris Masond1310b22008-01-24 16:13:08 -05003318 end = page_end;
Chris Mason40f76582014-05-21 13:35:51 -07003319 if (i_size <= start) {
Nikolay Borisovc6297322018-11-08 10:18:08 +02003320 btrfs_writepage_endio_finish_ordered(page, start, page_end, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003321 goto done;
3322 }
3323
Chris Masond1310b22008-01-24 16:13:08 -05003324 blocksize = inode->i_sb->s_blocksize;
3325
3326 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003327 u64 em_end;
David Sterba6273b7f2017-10-04 17:30:11 +02003328 u64 offset;
David Sterba58409ed2016-05-04 11:46:10 +02003329
Chris Mason40f76582014-05-21 13:35:51 -07003330 if (cur >= i_size) {
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02003331 btrfs_writepage_endio_finish_ordered(page, cur,
Nikolay Borisovc6297322018-11-08 10:18:08 +02003332 page_end, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003333 break;
3334 }
David Sterba3c98c622017-06-23 04:01:08 +02003335 em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003336 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003337 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003338 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003339 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003340 break;
3341 }
3342
3343 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003344 em_end = extent_map_end(em);
3345 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003346 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003347 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003348 iosize = ALIGN(iosize, blocksize);
David Sterba6273b7f2017-10-04 17:30:11 +02003349 offset = em->block_start + extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003350 bdev = em->bdev;
3351 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003352 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003353 free_extent_map(em);
3354 em = NULL;
3355
Chris Masonc8b97812008-10-29 14:49:59 -04003356 /*
3357 * compressed and inline extents are written through other
3358 * paths in the FS
3359 */
3360 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003361 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003362 /*
3363 * end_io notification does not happen here for
3364 * compressed extents
3365 */
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02003366 if (!compressed)
3367 btrfs_writepage_endio_finish_ordered(page, cur,
3368 cur + iosize - 1,
Nikolay Borisovc6297322018-11-08 10:18:08 +02003369 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003370 else if (compressed) {
3371 /* we don't want to end_page_writeback on
3372 * a compressed extent. this happens
3373 * elsewhere
3374 */
3375 nr++;
3376 }
3377
3378 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003379 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003380 continue;
3381 }
Chris Masonc8b97812008-10-29 14:49:59 -04003382
David Sterba5cdc84b2018-07-18 20:32:52 +02003383 btrfs_set_range_writeback(tree, cur, cur + iosize - 1);
David Sterba58409ed2016-05-04 11:46:10 +02003384 if (!PageWriteback(page)) {
3385 btrfs_err(BTRFS_I(inode)->root->fs_info,
3386 "page %lu not writeback, cur %llu end %llu",
3387 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003388 }
David Sterba58409ed2016-05-04 11:46:10 +02003389
David Sterba4b81ba42017-06-06 19:14:26 +02003390 ret = submit_extent_page(REQ_OP_WRITE | write_flags, tree, wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02003391 page, offset, iosize, pg_offset,
David Sterbac2df8bb2017-02-10 19:29:38 +01003392 bdev, &epd->bio,
David Sterba58409ed2016-05-04 11:46:10 +02003393 end_bio_extent_writepage,
3394 0, 0, 0, false);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003395 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003396 SetPageError(page);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003397 if (PageWriteback(page))
3398 end_page_writeback(page);
3399 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003400
Chris Masond1310b22008-01-24 16:13:08 -05003401 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003402 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003403 nr++;
3404 }
3405done:
Chris Mason40f76582014-05-21 13:35:51 -07003406 *nr_ret = nr;
Chris Mason40f76582014-05-21 13:35:51 -07003407 return ret;
3408}
3409
3410/*
3411 * the writepage semantics are similar to regular writepage. extent
3412 * records are inserted to lock ranges in the tree, and as dirty areas
3413 * are found, they are marked writeback. Then the lock bits are removed
3414 * and the end_io handler clears the writeback ranges
3415 */
3416static int __extent_writepage(struct page *page, struct writeback_control *wbc,
David Sterbaaab6e9e2017-11-30 18:00:02 +01003417 struct extent_page_data *epd)
Chris Mason40f76582014-05-21 13:35:51 -07003418{
3419 struct inode *inode = page->mapping->host;
Chris Mason40f76582014-05-21 13:35:51 -07003420 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003421 u64 page_end = start + PAGE_SIZE - 1;
Chris Mason40f76582014-05-21 13:35:51 -07003422 int ret;
3423 int nr = 0;
3424 size_t pg_offset = 0;
3425 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003426 unsigned long end_index = i_size >> PAGE_SHIFT;
David Sterbaf1c77c52017-06-06 19:03:49 +02003427 unsigned int write_flags = 0;
Chris Mason40f76582014-05-21 13:35:51 -07003428 unsigned long nr_written = 0;
3429
Liu Boff40adf2017-08-24 18:19:48 -06003430 write_flags = wbc_to_write_flags(wbc);
Chris Mason40f76582014-05-21 13:35:51 -07003431
3432 trace___extent_writepage(page, inode, wbc);
3433
3434 WARN_ON(!PageLocked(page));
3435
3436 ClearPageError(page);
3437
Johannes Thumshirn70730172018-12-05 15:23:03 +01003438 pg_offset = offset_in_page(i_size);
Chris Mason40f76582014-05-21 13:35:51 -07003439 if (page->index > end_index ||
3440 (page->index == end_index && !pg_offset)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003441 page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
Chris Mason40f76582014-05-21 13:35:51 -07003442 unlock_page(page);
3443 return 0;
3444 }
3445
3446 if (page->index == end_index) {
3447 char *userpage;
3448
3449 userpage = kmap_atomic(page);
3450 memset(userpage + pg_offset, 0,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003451 PAGE_SIZE - pg_offset);
Chris Mason40f76582014-05-21 13:35:51 -07003452 kunmap_atomic(userpage);
3453 flush_dcache_page(page);
3454 }
3455
3456 pg_offset = 0;
3457
3458 set_page_extent_mapped(page);
3459
Nikolay Borisov7789a552018-11-08 10:18:06 +02003460 if (!epd->extent_locked) {
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003461 ret = writepage_delalloc(inode, page, wbc, start, &nr_written);
Nikolay Borisov7789a552018-11-08 10:18:06 +02003462 if (ret == 1)
3463 goto done_unlocked;
3464 if (ret)
3465 goto done;
3466 }
Chris Mason40f76582014-05-21 13:35:51 -07003467
3468 ret = __extent_writepage_io(inode, page, wbc, epd,
3469 i_size, nr_written, write_flags, &nr);
3470 if (ret == 1)
3471 goto done_unlocked;
3472
3473done:
Chris Masond1310b22008-01-24 16:13:08 -05003474 if (nr == 0) {
3475 /* make sure the mapping tag for page dirty gets cleared */
3476 set_page_writeback(page);
3477 end_page_writeback(page);
3478 }
Filipe Manana61391d52014-05-09 17:17:40 +01003479 if (PageError(page)) {
3480 ret = ret < 0 ? ret : -EIO;
3481 end_extent_writepage(page, ret, start, page_end);
3482 }
Chris Masond1310b22008-01-24 16:13:08 -05003483 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003484 return ret;
Chris Mason771ed682008-11-06 22:02:51 -05003485
Chris Mason11c83492009-04-20 15:50:09 -04003486done_unlocked:
Chris Masond1310b22008-01-24 16:13:08 -05003487 return 0;
3488}
3489
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003490void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003491{
NeilBrown74316202014-07-07 15:16:04 +10003492 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3493 TASK_UNINTERRUPTIBLE);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003494}
3495
Chris Mason0e378df2014-05-19 20:55:27 -07003496static noinline_for_stack int
3497lock_extent_buffer_for_io(struct extent_buffer *eb,
3498 struct btrfs_fs_info *fs_info,
3499 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003500{
David Sterbacc5e31a2018-03-01 18:20:27 +01003501 int i, num_pages;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003502 int flush = 0;
3503 int ret = 0;
3504
3505 if (!btrfs_try_tree_write_lock(eb)) {
3506 flush = 1;
3507 flush_write_bio(epd);
3508 btrfs_tree_lock(eb);
3509 }
3510
3511 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3512 btrfs_tree_unlock(eb);
3513 if (!epd->sync_io)
3514 return 0;
3515 if (!flush) {
3516 flush_write_bio(epd);
3517 flush = 1;
3518 }
Chris Masona098d8e2012-03-21 12:09:56 -04003519 while (1) {
3520 wait_on_extent_buffer_writeback(eb);
3521 btrfs_tree_lock(eb);
3522 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3523 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003524 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003525 }
3526 }
3527
Josef Bacik51561ff2012-07-20 16:25:24 -04003528 /*
3529 * We need to do this to prevent races in people who check if the eb is
3530 * under IO since we can end up having no IO bits set for a short period
3531 * of time.
3532 */
3533 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003534 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3535 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003536 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003537 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Nikolay Borisov104b4e52017-06-20 21:01:20 +03003538 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
3539 -eb->len,
3540 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003541 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003542 } else {
3543 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003544 }
3545
3546 btrfs_tree_unlock(eb);
3547
3548 if (!ret)
3549 return ret;
3550
David Sterba65ad0102018-06-29 10:56:49 +02003551 num_pages = num_extent_pages(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003552 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003553 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003554
3555 if (!trylock_page(p)) {
3556 if (!flush) {
3557 flush_write_bio(epd);
3558 flush = 1;
3559 }
3560 lock_page(p);
3561 }
3562 }
3563
3564 return ret;
3565}
3566
3567static void end_extent_buffer_writeback(struct extent_buffer *eb)
3568{
3569 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003570 smp_mb__after_atomic();
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003571 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3572}
3573
Filipe Manana656f30d2014-09-26 12:25:56 +01003574static void set_btree_ioerr(struct page *page)
3575{
3576 struct extent_buffer *eb = (struct extent_buffer *)page->private;
Filipe Manana656f30d2014-09-26 12:25:56 +01003577
3578 SetPageError(page);
3579 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3580 return;
3581
3582 /*
3583 * If writeback for a btree extent that doesn't belong to a log tree
3584 * failed, increment the counter transaction->eb_write_errors.
3585 * We do this because while the transaction is running and before it's
3586 * committing (when we call filemap_fdata[write|wait]_range against
3587 * the btree inode), we might have
3588 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3589 * returns an error or an error happens during writeback, when we're
3590 * committing the transaction we wouldn't know about it, since the pages
3591 * can be no longer dirty nor marked anymore for writeback (if a
3592 * subsequent modification to the extent buffer didn't happen before the
3593 * transaction commit), which makes filemap_fdata[write|wait]_range not
3594 * able to find the pages tagged with SetPageError at transaction
3595 * commit time. So if this happens we must abort the transaction,
3596 * otherwise we commit a super block with btree roots that point to
3597 * btree nodes/leafs whose content on disk is invalid - either garbage
3598 * or the content of some node/leaf from a past generation that got
3599 * cowed or deleted and is no longer valid.
3600 *
3601 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3602 * not be enough - we need to distinguish between log tree extents vs
3603 * non-log tree extents, and the next filemap_fdatawait_range() call
3604 * will catch and clear such errors in the mapping - and that call might
3605 * be from a log sync and not from a transaction commit. Also, checking
3606 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3607 * not done and would not be reliable - the eb might have been released
3608 * from memory and reading it back again means that flag would not be
3609 * set (since it's a runtime flag, not persisted on disk).
3610 *
3611 * Using the flags below in the btree inode also makes us achieve the
3612 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3613 * writeback for all dirty pages and before filemap_fdatawait_range()
3614 * is called, the writeback for all dirty pages had already finished
3615 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3616 * filemap_fdatawait_range() would return success, as it could not know
3617 * that writeback errors happened (the pages were no longer tagged for
3618 * writeback).
3619 */
3620 switch (eb->log_index) {
3621 case -1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003622 set_bit(BTRFS_FS_BTREE_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003623 break;
3624 case 0:
Josef Bacikafcdd122016-09-02 15:40:02 -04003625 set_bit(BTRFS_FS_LOG1_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003626 break;
3627 case 1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003628 set_bit(BTRFS_FS_LOG2_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003629 break;
3630 default:
3631 BUG(); /* unexpected, logic error */
3632 }
3633}
3634
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003635static void end_bio_extent_buffer_writepage(struct bio *bio)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003636{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003637 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003638 struct extent_buffer *eb;
Kent Overstreet2c30c712013-11-07 12:20:26 -08003639 int i, done;
Ming Lei6dc4f102019-02-15 19:13:19 +08003640 struct bvec_iter_all iter_all;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003641
David Sterbac09abff2017-07-13 18:10:07 +02003642 ASSERT(!bio_flagged(bio, BIO_CLONED));
Ming Lei6dc4f102019-02-15 19:13:19 +08003643 bio_for_each_segment_all(bvec, bio, i, iter_all) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003644 struct page *page = bvec->bv_page;
3645
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003646 eb = (struct extent_buffer *)page->private;
3647 BUG_ON(!eb);
3648 done = atomic_dec_and_test(&eb->io_pages);
3649
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02003650 if (bio->bi_status ||
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003651 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003652 ClearPageUptodate(page);
Filipe Manana656f30d2014-09-26 12:25:56 +01003653 set_btree_ioerr(page);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003654 }
3655
3656 end_page_writeback(page);
3657
3658 if (!done)
3659 continue;
3660
3661 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003662 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003663
3664 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003665}
3666
Chris Mason0e378df2014-05-19 20:55:27 -07003667static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003668 struct btrfs_fs_info *fs_info,
3669 struct writeback_control *wbc,
3670 struct extent_page_data *epd)
3671{
3672 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
Josef Bacikf28491e2013-12-16 13:24:27 -05003673 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003674 u64 offset = eb->start;
Liu Bo851cd172016-09-23 13:44:44 -07003675 u32 nritems;
David Sterbacc5e31a2018-03-01 18:20:27 +01003676 int i, num_pages;
Liu Bo851cd172016-09-23 13:44:44 -07003677 unsigned long start, end;
Liu Boff40adf2017-08-24 18:19:48 -06003678 unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003679 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003680
Filipe Manana656f30d2014-09-26 12:25:56 +01003681 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02003682 num_pages = num_extent_pages(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003683 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003684
Liu Bo851cd172016-09-23 13:44:44 -07003685 /* set btree blocks beyond nritems with 0 to avoid stale content. */
3686 nritems = btrfs_header_nritems(eb);
Liu Bo3eb548e2016-09-14 17:22:57 -07003687 if (btrfs_header_level(eb) > 0) {
Liu Bo3eb548e2016-09-14 17:22:57 -07003688 end = btrfs_node_key_ptr_offset(nritems);
3689
David Sterbab159fa22016-11-08 18:09:03 +01003690 memzero_extent_buffer(eb, end, eb->len - end);
Liu Bo851cd172016-09-23 13:44:44 -07003691 } else {
3692 /*
3693 * leaf:
3694 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
3695 */
3696 start = btrfs_item_nr_offset(nritems);
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003697 end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(fs_info, eb);
David Sterbab159fa22016-11-08 18:09:03 +01003698 memzero_extent_buffer(eb, start, end - start);
Liu Bo3eb548e2016-09-14 17:22:57 -07003699 }
3700
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003701 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003702 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003703
3704 clear_page_dirty_for_io(p);
3705 set_page_writeback(p);
David Sterba4b81ba42017-06-06 19:14:26 +02003706 ret = submit_extent_page(REQ_OP_WRITE | write_flags, tree, wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02003707 p, offset, PAGE_SIZE, 0, bdev,
David Sterbac2df8bb2017-02-10 19:29:38 +01003708 &epd->bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003709 end_bio_extent_buffer_writepage,
Liu Bo18fdc672017-09-13 12:18:22 -06003710 0, 0, 0, false);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003711 if (ret) {
Filipe Manana656f30d2014-09-26 12:25:56 +01003712 set_btree_ioerr(p);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003713 if (PageWriteback(p))
3714 end_page_writeback(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003715 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3716 end_extent_buffer_writeback(eb);
3717 ret = -EIO;
3718 break;
3719 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003720 offset += PAGE_SIZE;
David Sterba3d4b9492017-02-10 19:33:41 +01003721 update_nr_written(wbc, 1);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003722 unlock_page(p);
3723 }
3724
3725 if (unlikely(ret)) {
3726 for (; i < num_pages; i++) {
Chris Masonbbf65cf2014-10-04 09:56:45 -07003727 struct page *p = eb->pages[i];
Liu Bo81465022014-09-23 22:22:33 +08003728 clear_page_dirty_for_io(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003729 unlock_page(p);
3730 }
3731 }
3732
3733 return ret;
3734}
3735
3736int btree_write_cache_pages(struct address_space *mapping,
3737 struct writeback_control *wbc)
3738{
3739 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3740 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3741 struct extent_buffer *eb, *prev_eb = NULL;
3742 struct extent_page_data epd = {
3743 .bio = NULL,
3744 .tree = tree,
3745 .extent_locked = 0,
3746 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3747 };
3748 int ret = 0;
3749 int done = 0;
3750 int nr_to_write_done = 0;
3751 struct pagevec pvec;
3752 int nr_pages;
3753 pgoff_t index;
3754 pgoff_t end; /* Inclusive */
3755 int scanned = 0;
Matthew Wilcox10bbd232017-12-05 17:30:38 -05003756 xa_mark_t tag;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003757
Mel Gorman86679822017-11-15 17:37:52 -08003758 pagevec_init(&pvec);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003759 if (wbc->range_cyclic) {
3760 index = mapping->writeback_index; /* Start from prev offset */
3761 end = -1;
3762 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003763 index = wbc->range_start >> PAGE_SHIFT;
3764 end = wbc->range_end >> PAGE_SHIFT;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003765 scanned = 1;
3766 }
3767 if (wbc->sync_mode == WB_SYNC_ALL)
3768 tag = PAGECACHE_TAG_TOWRITE;
3769 else
3770 tag = PAGECACHE_TAG_DIRTY;
3771retry:
3772 if (wbc->sync_mode == WB_SYNC_ALL)
3773 tag_pages_for_writeback(mapping, index, end);
3774 while (!done && !nr_to_write_done && (index <= end) &&
Jan Kara4006f432017-11-15 17:34:37 -08003775 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
Jan Kara67fd7072017-11-15 17:35:19 -08003776 tag))) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003777 unsigned i;
3778
3779 scanned = 1;
3780 for (i = 0; i < nr_pages; i++) {
3781 struct page *page = pvec.pages[i];
3782
3783 if (!PagePrivate(page))
3784 continue;
3785
Josef Bacikb5bae262012-09-14 13:43:01 -04003786 spin_lock(&mapping->private_lock);
3787 if (!PagePrivate(page)) {
3788 spin_unlock(&mapping->private_lock);
3789 continue;
3790 }
3791
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003792 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003793
3794 /*
3795 * Shouldn't happen and normally this would be a BUG_ON
3796 * but no sense in crashing the users box for something
3797 * we can survive anyway.
3798 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303799 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003800 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003801 continue;
3802 }
3803
Josef Bacikb5bae262012-09-14 13:43:01 -04003804 if (eb == prev_eb) {
3805 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003806 continue;
3807 }
3808
Josef Bacikb5bae262012-09-14 13:43:01 -04003809 ret = atomic_inc_not_zero(&eb->refs);
3810 spin_unlock(&mapping->private_lock);
3811 if (!ret)
3812 continue;
3813
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003814 prev_eb = eb;
3815 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3816 if (!ret) {
3817 free_extent_buffer(eb);
3818 continue;
3819 }
3820
3821 ret = write_one_eb(eb, fs_info, wbc, &epd);
3822 if (ret) {
3823 done = 1;
3824 free_extent_buffer(eb);
3825 break;
3826 }
3827 free_extent_buffer(eb);
3828
3829 /*
3830 * the filesystem may choose to bump up nr_to_write.
3831 * We have to make sure to honor the new nr_to_write
3832 * at any time
3833 */
3834 nr_to_write_done = wbc->nr_to_write <= 0;
3835 }
3836 pagevec_release(&pvec);
3837 cond_resched();
3838 }
3839 if (!scanned && !done) {
3840 /*
3841 * We hit the last page and there is more work to be done: wrap
3842 * back to the start of the file
3843 */
3844 scanned = 1;
3845 index = 0;
3846 goto retry;
3847 }
3848 flush_write_bio(&epd);
3849 return ret;
3850}
3851
Chris Masond1310b22008-01-24 16:13:08 -05003852/**
Chris Mason4bef0842008-09-08 11:18:08 -04003853 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
Chris Masond1310b22008-01-24 16:13:08 -05003854 * @mapping: address space structure to write
3855 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
David Sterba935db852017-06-23 04:30:28 +02003856 * @data: data passed to __extent_writepage function
Chris Masond1310b22008-01-24 16:13:08 -05003857 *
3858 * If a page is already under I/O, write_cache_pages() skips it, even
3859 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3860 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3861 * and msync() need to guarantee that all the data which was dirty at the time
3862 * the call was made get new I/O started against them. If wbc->sync_mode is
3863 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3864 * existing IO to complete.
3865 */
David Sterba4242b642017-02-10 19:38:24 +01003866static int extent_write_cache_pages(struct address_space *mapping,
Chris Mason4bef0842008-09-08 11:18:08 -04003867 struct writeback_control *wbc,
David Sterbaaab6e9e2017-11-30 18:00:02 +01003868 struct extent_page_data *epd)
Chris Masond1310b22008-01-24 16:13:08 -05003869{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003870 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003871 int ret = 0;
3872 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003873 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003874 struct pagevec pvec;
3875 int nr_pages;
3876 pgoff_t index;
3877 pgoff_t end; /* Inclusive */
Liu Boa91326672016-03-07 16:56:21 -08003878 pgoff_t done_index;
3879 int range_whole = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003880 int scanned = 0;
Matthew Wilcox10bbd232017-12-05 17:30:38 -05003881 xa_mark_t tag;
Chris Masond1310b22008-01-24 16:13:08 -05003882
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003883 /*
3884 * We have to hold onto the inode so that ordered extents can do their
3885 * work when the IO finishes. The alternative to this is failing to add
3886 * an ordered extent if the igrab() fails there and that is a huge pain
3887 * to deal with, so instead just hold onto the inode throughout the
3888 * writepages operation. If it fails here we are freeing up the inode
3889 * anyway and we'd rather not waste our time writing out stuff that is
3890 * going to be truncated anyway.
3891 */
3892 if (!igrab(inode))
3893 return 0;
3894
Mel Gorman86679822017-11-15 17:37:52 -08003895 pagevec_init(&pvec);
Chris Masond1310b22008-01-24 16:13:08 -05003896 if (wbc->range_cyclic) {
3897 index = mapping->writeback_index; /* Start from prev offset */
3898 end = -1;
3899 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003900 index = wbc->range_start >> PAGE_SHIFT;
3901 end = wbc->range_end >> PAGE_SHIFT;
Liu Boa91326672016-03-07 16:56:21 -08003902 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
3903 range_whole = 1;
Chris Masond1310b22008-01-24 16:13:08 -05003904 scanned = 1;
3905 }
Ethan Lien3cd24c62018-11-01 14:49:03 +08003906
3907 /*
3908 * We do the tagged writepage as long as the snapshot flush bit is set
3909 * and we are the first one who do the filemap_flush() on this inode.
3910 *
3911 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
3912 * not race in and drop the bit.
3913 */
3914 if (range_whole && wbc->nr_to_write == LONG_MAX &&
3915 test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
3916 &BTRFS_I(inode)->runtime_flags))
3917 wbc->tagged_writepages = 1;
3918
3919 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Josef Bacikf7aaa062011-07-15 21:26:38 +00003920 tag = PAGECACHE_TAG_TOWRITE;
3921 else
3922 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003923retry:
Ethan Lien3cd24c62018-11-01 14:49:03 +08003924 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Josef Bacikf7aaa062011-07-15 21:26:38 +00003925 tag_pages_for_writeback(mapping, index, end);
Liu Boa91326672016-03-07 16:56:21 -08003926 done_index = index;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003927 while (!done && !nr_to_write_done && (index <= end) &&
Jan Kara67fd7072017-11-15 17:35:19 -08003928 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping,
3929 &index, end, tag))) {
Chris Masond1310b22008-01-24 16:13:08 -05003930 unsigned i;
3931
3932 scanned = 1;
3933 for (i = 0; i < nr_pages; i++) {
3934 struct page *page = pvec.pages[i];
3935
Liu Boa91326672016-03-07 16:56:21 -08003936 done_index = page->index;
Chris Masond1310b22008-01-24 16:13:08 -05003937 /*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07003938 * At this point we hold neither the i_pages lock nor
3939 * the page lock: the page may be truncated or
3940 * invalidated (changing page->mapping to NULL),
3941 * or even swizzled back from swapper_space to
3942 * tmpfs file mapping
Chris Masond1310b22008-01-24 16:13:08 -05003943 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05003944 if (!trylock_page(page)) {
David Sterbaaab6e9e2017-11-30 18:00:02 +01003945 flush_write_bio(epd);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003946 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04003947 }
Chris Masond1310b22008-01-24 16:13:08 -05003948
3949 if (unlikely(page->mapping != mapping)) {
3950 unlock_page(page);
3951 continue;
3952 }
3953
Chris Masond2c3f4f2008-11-19 12:44:22 -05003954 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05003955 if (PageWriteback(page))
David Sterbaaab6e9e2017-11-30 18:00:02 +01003956 flush_write_bio(epd);
Chris Masond1310b22008-01-24 16:13:08 -05003957 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003958 }
Chris Masond1310b22008-01-24 16:13:08 -05003959
3960 if (PageWriteback(page) ||
3961 !clear_page_dirty_for_io(page)) {
3962 unlock_page(page);
3963 continue;
3964 }
3965
David Sterbaaab6e9e2017-11-30 18:00:02 +01003966 ret = __extent_writepage(page, wbc, epd);
Chris Masond1310b22008-01-24 16:13:08 -05003967
3968 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3969 unlock_page(page);
3970 ret = 0;
3971 }
Liu Boa91326672016-03-07 16:56:21 -08003972 if (ret < 0) {
3973 /*
3974 * done_index is set past this page,
3975 * so media errors will not choke
3976 * background writeout for the entire
3977 * file. This has consequences for
3978 * range_cyclic semantics (ie. it may
3979 * not be suitable for data integrity
3980 * writeout).
3981 */
3982 done_index = page->index + 1;
3983 done = 1;
3984 break;
3985 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003986
3987 /*
3988 * the filesystem may choose to bump up nr_to_write.
3989 * We have to make sure to honor the new nr_to_write
3990 * at any time
3991 */
3992 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05003993 }
3994 pagevec_release(&pvec);
3995 cond_resched();
3996 }
Liu Bo894b36e2016-03-07 16:56:22 -08003997 if (!scanned && !done) {
Chris Masond1310b22008-01-24 16:13:08 -05003998 /*
3999 * We hit the last page and there is more work to be done: wrap
4000 * back to the start of the file
4001 */
4002 scanned = 1;
4003 index = 0;
4004 goto retry;
4005 }
Liu Boa91326672016-03-07 16:56:21 -08004006
4007 if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
4008 mapping->writeback_index = done_index;
4009
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004010 btrfs_add_delayed_iput(inode);
Liu Bo894b36e2016-03-07 16:56:22 -08004011 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05004012}
Chris Masond1310b22008-01-24 16:13:08 -05004013
David Sterbaaab6e9e2017-11-30 18:00:02 +01004014static void flush_write_bio(struct extent_page_data *epd)
Chris Masonffbd5172009-04-20 15:50:09 -04004015{
4016 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04004017 int ret;
4018
Liu Bo18fdc672017-09-13 12:18:22 -06004019 ret = submit_one_bio(epd->bio, 0, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004020 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04004021 epd->bio = NULL;
4022 }
4023}
4024
Nikolay Borisov0a9b0e52017-12-08 15:55:59 +02004025int extent_write_full_page(struct page *page, struct writeback_control *wbc)
Chris Masond1310b22008-01-24 16:13:08 -05004026{
4027 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004028 struct extent_page_data epd = {
4029 .bio = NULL,
Nikolay Borisov0a9b0e52017-12-08 15:55:59 +02004030 .tree = &BTRFS_I(page->mapping->host)->io_tree,
Chris Mason771ed682008-11-06 22:02:51 -05004031 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004032 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05004033 };
Chris Masond1310b22008-01-24 16:13:08 -05004034
Chris Masond1310b22008-01-24 16:13:08 -05004035 ret = __extent_writepage(page, wbc, &epd);
4036
David Sterbae2932ee2017-06-23 04:16:17 +02004037 flush_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004038 return ret;
4039}
Chris Masond1310b22008-01-24 16:13:08 -05004040
Nikolay Borisov5e3ee232017-12-08 15:55:58 +02004041int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
Chris Mason771ed682008-11-06 22:02:51 -05004042 int mode)
4043{
4044 int ret = 0;
4045 struct address_space *mapping = inode->i_mapping;
Nikolay Borisov5e3ee232017-12-08 15:55:58 +02004046 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Chris Mason771ed682008-11-06 22:02:51 -05004047 struct page *page;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004048 unsigned long nr_pages = (end - start + PAGE_SIZE) >>
4049 PAGE_SHIFT;
Chris Mason771ed682008-11-06 22:02:51 -05004050
4051 struct extent_page_data epd = {
4052 .bio = NULL,
4053 .tree = tree,
Chris Mason771ed682008-11-06 22:02:51 -05004054 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04004055 .sync_io = mode == WB_SYNC_ALL,
Chris Mason771ed682008-11-06 22:02:51 -05004056 };
4057 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05004058 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05004059 .nr_to_write = nr_pages * 2,
4060 .range_start = start,
4061 .range_end = end + 1,
4062 };
4063
Chris Masond3977122009-01-05 21:25:51 -05004064 while (start <= end) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004065 page = find_get_page(mapping, start >> PAGE_SHIFT);
Chris Mason771ed682008-11-06 22:02:51 -05004066 if (clear_page_dirty_for_io(page))
4067 ret = __extent_writepage(page, &wbc_writepages, &epd);
4068 else {
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02004069 btrfs_writepage_endio_finish_ordered(page, start,
Nikolay Borisovc6297322018-11-08 10:18:08 +02004070 start + PAGE_SIZE - 1, 1);
Chris Mason771ed682008-11-06 22:02:51 -05004071 unlock_page(page);
4072 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004073 put_page(page);
4074 start += PAGE_SIZE;
Chris Mason771ed682008-11-06 22:02:51 -05004075 }
4076
David Sterbae2932ee2017-06-23 04:16:17 +02004077 flush_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05004078 return ret;
4079}
Chris Masond1310b22008-01-24 16:13:08 -05004080
Nikolay Borisov8ae225a2018-04-19 10:46:38 +03004081int extent_writepages(struct address_space *mapping,
Chris Masond1310b22008-01-24 16:13:08 -05004082 struct writeback_control *wbc)
4083{
4084 int ret = 0;
4085 struct extent_page_data epd = {
4086 .bio = NULL,
Nikolay Borisov8ae225a2018-04-19 10:46:38 +03004087 .tree = &BTRFS_I(mapping->host)->io_tree,
Chris Mason771ed682008-11-06 22:02:51 -05004088 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004089 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05004090 };
4091
David Sterba935db852017-06-23 04:30:28 +02004092 ret = extent_write_cache_pages(mapping, wbc, &epd);
David Sterbae2932ee2017-06-23 04:16:17 +02004093 flush_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004094 return ret;
4095}
Chris Masond1310b22008-01-24 16:13:08 -05004096
Nikolay Borisov2a3ff0a2018-04-19 10:46:36 +03004097int extent_readpages(struct address_space *mapping, struct list_head *pages,
4098 unsigned nr_pages)
Chris Masond1310b22008-01-24 16:13:08 -05004099{
4100 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04004101 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06004102 struct page *pagepool[16];
Miao Xie125bac012013-07-25 19:22:37 +08004103 struct extent_map *em_cached = NULL;
Nikolay Borisov2a3ff0a2018-04-19 10:46:36 +03004104 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
Liu Bo67c96842012-07-20 21:43:09 -06004105 int nr = 0;
Filipe Manana808f80b2015-09-28 09:56:26 +01004106 u64 prev_em_start = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05004107
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004108 while (!list_empty(pages)) {
4109 for (nr = 0; nr < ARRAY_SIZE(pagepool) && !list_empty(pages);) {
Nikolay Borisovf86196e2019-01-03 15:29:02 -08004110 struct page *page = lru_to_page(pages);
Chris Masond1310b22008-01-24 16:13:08 -05004111
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004112 prefetchw(&page->flags);
4113 list_del(&page->lru);
4114 if (add_to_page_cache_lru(page, mapping, page->index,
4115 readahead_gfp_mask(mapping))) {
4116 put_page(page);
4117 continue;
4118 }
4119
4120 pagepool[nr++] = page;
Chris Masond1310b22008-01-24 16:13:08 -05004121 }
Liu Bo67c96842012-07-20 21:43:09 -06004122
David Sterbae4d17ef2017-06-23 04:09:57 +02004123 __extent_readpages(tree, pagepool, nr, &em_cached, &bio,
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004124 &bio_flags, &prev_em_start);
Chris Masond1310b22008-01-24 16:13:08 -05004125 }
Liu Bo67c96842012-07-20 21:43:09 -06004126
Miao Xie125bac012013-07-25 19:22:37 +08004127 if (em_cached)
4128 free_extent_map(em_cached);
4129
Chris Masond1310b22008-01-24 16:13:08 -05004130 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05004131 return submit_one_bio(bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004132 return 0;
4133}
Chris Masond1310b22008-01-24 16:13:08 -05004134
4135/*
4136 * basic invalidatepage code, this waits on any locked or writeback
4137 * ranges corresponding to the page, and then deletes any extent state
4138 * records from the tree
4139 */
4140int extent_invalidatepage(struct extent_io_tree *tree,
4141 struct page *page, unsigned long offset)
4142{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004143 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004144 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004145 u64 end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05004146 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4147
Qu Wenruofda28322013-02-26 08:10:22 +00004148 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004149 if (start > end)
4150 return 0;
4151
David Sterbaff13db42015-12-03 14:30:40 +01004152 lock_extent_bits(tree, start, end, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004153 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05004154 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04004155 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4156 EXTENT_DO_ACCOUNTING,
David Sterbaae0f1622017-10-31 16:37:52 +01004157 1, 1, &cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05004158 return 0;
4159}
Chris Masond1310b22008-01-24 16:13:08 -05004160
4161/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004162 * a helper for releasepage, this tests for areas of the page that
4163 * are locked or under IO and drops the related state bits if it is safe
4164 * to drop the page.
4165 */
Nikolay Borisov29c68b2d2018-04-19 10:46:35 +03004166static int try_release_extent_state(struct extent_io_tree *tree,
Eric Sandeen48a3b632013-04-25 20:41:01 +00004167 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004168{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004169 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004170 u64 end = start + PAGE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004171 int ret = 1;
4172
Chris Mason211f90e2008-07-18 11:56:15 -04004173 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04004174 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04004175 ret = 0;
4176 else {
Chris Mason11ef1602009-09-23 20:28:46 -04004177 /*
4178 * at this point we can safely clear everything except the
4179 * locked bit and the nodatasum bit
4180 */
David Sterba66b0c882017-10-31 16:30:47 +01004181 ret = __clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004182 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
David Sterba66b0c882017-10-31 16:30:47 +01004183 0, 0, NULL, mask, NULL);
Chris Masone3f24cc2011-02-14 12:52:08 -05004184
4185 /* if clear_extent_bit failed for enomem reasons,
4186 * we can't allow the release to continue.
4187 */
4188 if (ret < 0)
4189 ret = 0;
4190 else
4191 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004192 }
4193 return ret;
4194}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004195
4196/*
Chris Masond1310b22008-01-24 16:13:08 -05004197 * a helper for releasepage. As long as there are no locked extents
4198 * in the range corresponding to the page, both state records and extent
4199 * map records are removed
4200 */
Nikolay Borisov477a30b2018-04-19 10:46:34 +03004201int try_release_extent_mapping(struct page *page, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004202{
4203 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004204 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004205 u64 end = start + PAGE_SIZE - 1;
Filipe Mananabd3599a2018-07-12 01:36:43 +01004206 struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
4207 struct extent_io_tree *tree = &btrfs_inode->io_tree;
4208 struct extent_map_tree *map = &btrfs_inode->extent_tree;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004209
Mel Gormand0164ad2015-11-06 16:28:21 -08004210 if (gfpflags_allow_blocking(mask) &&
Byongho Leeee221842015-12-15 01:42:10 +09004211 page->mapping->host->i_size > SZ_16M) {
Yan39b56372008-02-15 10:40:50 -05004212 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004213 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004214 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004215 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004216 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004217 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004218 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004219 break;
4220 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004221 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4222 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004223 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004224 free_extent_map(em);
4225 break;
4226 }
4227 if (!test_range_bit(tree, em->start,
4228 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04004229 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04004230 0, NULL)) {
Filipe Mananabd3599a2018-07-12 01:36:43 +01004231 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4232 &btrfs_inode->runtime_flags);
Chris Mason70dec802008-01-29 09:59:12 -05004233 remove_extent_mapping(map, em);
4234 /* once for the rb tree */
4235 free_extent_map(em);
4236 }
4237 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004238 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004239
4240 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004241 free_extent_map(em);
4242 }
Chris Masond1310b22008-01-24 16:13:08 -05004243 }
Nikolay Borisov29c68b2d2018-04-19 10:46:35 +03004244 return try_release_extent_state(tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004245}
Chris Masond1310b22008-01-24 16:13:08 -05004246
Chris Masonec29ed52011-02-23 16:23:20 -05004247/*
4248 * helper function for fiemap, which doesn't want to see any holes.
4249 * This maps until we find something past 'last'
4250 */
4251static struct extent_map *get_extent_skip_holes(struct inode *inode,
David Sterbae3350e12017-06-23 04:09:57 +02004252 u64 offset, u64 last)
Chris Masonec29ed52011-02-23 16:23:20 -05004253{
Jeff Mahoneyda170662016-06-15 09:22:56 -04004254 u64 sectorsize = btrfs_inode_sectorsize(inode);
Chris Masonec29ed52011-02-23 16:23:20 -05004255 struct extent_map *em;
4256 u64 len;
4257
4258 if (offset >= last)
4259 return NULL;
4260
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304261 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004262 len = last - offset;
4263 if (len == 0)
4264 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004265 len = ALIGN(len, sectorsize);
David Sterbae3350e12017-06-23 04:09:57 +02004266 em = btrfs_get_extent_fiemap(BTRFS_I(inode), NULL, 0, offset,
4267 len, 0);
David Sterbac7040052011-04-19 18:00:01 +02004268 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004269 return em;
4270
4271 /* if this isn't a hole return it */
Nikolay Borisov4a2d25c2017-11-23 10:51:43 +02004272 if (em->block_start != EXTENT_MAP_HOLE)
Chris Masonec29ed52011-02-23 16:23:20 -05004273 return em;
Chris Masonec29ed52011-02-23 16:23:20 -05004274
4275 /* this is a hole, advance to the next extent */
4276 offset = extent_map_end(em);
4277 free_extent_map(em);
4278 if (offset >= last)
4279 break;
4280 }
4281 return NULL;
4282}
4283
Qu Wenruo47518322017-04-07 10:43:15 +08004284/*
4285 * To cache previous fiemap extent
4286 *
4287 * Will be used for merging fiemap extent
4288 */
4289struct fiemap_cache {
4290 u64 offset;
4291 u64 phys;
4292 u64 len;
4293 u32 flags;
4294 bool cached;
4295};
4296
4297/*
4298 * Helper to submit fiemap extent.
4299 *
4300 * Will try to merge current fiemap extent specified by @offset, @phys,
4301 * @len and @flags with cached one.
4302 * And only when we fails to merge, cached one will be submitted as
4303 * fiemap extent.
4304 *
4305 * Return value is the same as fiemap_fill_next_extent().
4306 */
4307static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
4308 struct fiemap_cache *cache,
4309 u64 offset, u64 phys, u64 len, u32 flags)
4310{
4311 int ret = 0;
4312
4313 if (!cache->cached)
4314 goto assign;
4315
4316 /*
4317 * Sanity check, extent_fiemap() should have ensured that new
Andrea Gelmini52042d82018-11-28 12:05:13 +01004318 * fiemap extent won't overlap with cached one.
Qu Wenruo47518322017-04-07 10:43:15 +08004319 * Not recoverable.
4320 *
4321 * NOTE: Physical address can overlap, due to compression
4322 */
4323 if (cache->offset + cache->len > offset) {
4324 WARN_ON(1);
4325 return -EINVAL;
4326 }
4327
4328 /*
4329 * Only merges fiemap extents if
4330 * 1) Their logical addresses are continuous
4331 *
4332 * 2) Their physical addresses are continuous
4333 * So truly compressed (physical size smaller than logical size)
4334 * extents won't get merged with each other
4335 *
4336 * 3) Share same flags except FIEMAP_EXTENT_LAST
4337 * So regular extent won't get merged with prealloc extent
4338 */
4339 if (cache->offset + cache->len == offset &&
4340 cache->phys + cache->len == phys &&
4341 (cache->flags & ~FIEMAP_EXTENT_LAST) ==
4342 (flags & ~FIEMAP_EXTENT_LAST)) {
4343 cache->len += len;
4344 cache->flags |= flags;
4345 goto try_submit_last;
4346 }
4347
4348 /* Not mergeable, need to submit cached one */
4349 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4350 cache->len, cache->flags);
4351 cache->cached = false;
4352 if (ret)
4353 return ret;
4354assign:
4355 cache->cached = true;
4356 cache->offset = offset;
4357 cache->phys = phys;
4358 cache->len = len;
4359 cache->flags = flags;
4360try_submit_last:
4361 if (cache->flags & FIEMAP_EXTENT_LAST) {
4362 ret = fiemap_fill_next_extent(fieinfo, cache->offset,
4363 cache->phys, cache->len, cache->flags);
4364 cache->cached = false;
4365 }
4366 return ret;
4367}
4368
4369/*
Qu Wenruo848c23b2017-06-22 10:01:21 +08004370 * Emit last fiemap cache
Qu Wenruo47518322017-04-07 10:43:15 +08004371 *
Qu Wenruo848c23b2017-06-22 10:01:21 +08004372 * The last fiemap cache may still be cached in the following case:
4373 * 0 4k 8k
4374 * |<- Fiemap range ->|
4375 * |<------------ First extent ----------->|
4376 *
4377 * In this case, the first extent range will be cached but not emitted.
4378 * So we must emit it before ending extent_fiemap().
Qu Wenruo47518322017-04-07 10:43:15 +08004379 */
Qu Wenruo848c23b2017-06-22 10:01:21 +08004380static int emit_last_fiemap_cache(struct btrfs_fs_info *fs_info,
4381 struct fiemap_extent_info *fieinfo,
4382 struct fiemap_cache *cache)
Qu Wenruo47518322017-04-07 10:43:15 +08004383{
4384 int ret;
4385
4386 if (!cache->cached)
4387 return 0;
4388
Qu Wenruo47518322017-04-07 10:43:15 +08004389 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4390 cache->len, cache->flags);
4391 cache->cached = false;
4392 if (ret > 0)
4393 ret = 0;
4394 return ret;
4395}
4396
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004397int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
David Sterba2135fb92017-06-23 04:09:57 +02004398 __u64 start, __u64 len)
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004399{
Josef Bacik975f84f2010-11-23 19:36:57 +00004400 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004401 u64 off = start;
4402 u64 max = start + len;
4403 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004404 u32 found_type;
4405 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004406 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004407 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004408 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004409 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004410 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004411 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004412 struct btrfs_path *path;
Josef Bacikdc046b12014-09-10 16:20:45 -04004413 struct btrfs_root *root = BTRFS_I(inode)->root;
Qu Wenruo47518322017-04-07 10:43:15 +08004414 struct fiemap_cache cache = { 0 };
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004415 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004416 u64 em_start = 0;
4417 u64 em_len = 0;
4418 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004419
4420 if (len == 0)
4421 return -EINVAL;
4422
Josef Bacik975f84f2010-11-23 19:36:57 +00004423 path = btrfs_alloc_path();
4424 if (!path)
4425 return -ENOMEM;
4426 path->leave_spinning = 1;
4427
Jeff Mahoneyda170662016-06-15 09:22:56 -04004428 start = round_down(start, btrfs_inode_sectorsize(inode));
4429 len = round_up(max, btrfs_inode_sectorsize(inode)) - start;
Josef Bacik4d479cf2011-11-17 11:34:31 -05004430
Chris Masonec29ed52011-02-23 16:23:20 -05004431 /*
4432 * lookup the last file extent. We're not using i_size here
4433 * because there might be preallocation past i_size
4434 */
David Sterbaf85b7372017-01-20 14:54:07 +01004435 ret = btrfs_lookup_file_extent(NULL, root, path,
4436 btrfs_ino(BTRFS_I(inode)), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004437 if (ret < 0) {
4438 btrfs_free_path(path);
4439 return ret;
Liu Bo2d324f52016-05-17 17:21:48 -07004440 } else {
4441 WARN_ON(!ret);
4442 if (ret == 1)
4443 ret = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004444 }
Liu Bo2d324f52016-05-17 17:21:48 -07004445
Josef Bacik975f84f2010-11-23 19:36:57 +00004446 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004447 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
David Sterba962a2982014-06-04 18:41:45 +02004448 found_type = found_key.type;
Josef Bacik975f84f2010-11-23 19:36:57 +00004449
Chris Masonec29ed52011-02-23 16:23:20 -05004450 /* No extents, but there might be delalloc bits */
Nikolay Borisov4a0cc7c2017-01-10 20:35:31 +02004451 if (found_key.objectid != btrfs_ino(BTRFS_I(inode)) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004452 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004453 /* have to trust i_size as the end */
4454 last = (u64)-1;
4455 last_for_get_extent = isize;
4456 } else {
4457 /*
4458 * remember the start of the last extent. There are a
4459 * bunch of different factors that go into the length of the
4460 * extent, so its much less complex to remember where it started
4461 */
4462 last = found_key.offset;
4463 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004464 }
Liu Bofe09e162013-09-22 12:54:23 +08004465 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004466
Chris Masonec29ed52011-02-23 16:23:20 -05004467 /*
4468 * we might have some extents allocated but more delalloc past those
4469 * extents. so, we trust isize unless the start of the last extent is
4470 * beyond isize
4471 */
4472 if (last < isize) {
4473 last = (u64)-1;
4474 last_for_get_extent = isize;
4475 }
4476
David Sterbaff13db42015-12-03 14:30:40 +01004477 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004478 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004479
David Sterbae3350e12017-06-23 04:09:57 +02004480 em = get_extent_skip_holes(inode, start, last_for_get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004481 if (!em)
4482 goto out;
4483 if (IS_ERR(em)) {
4484 ret = PTR_ERR(em);
4485 goto out;
4486 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004487
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004488 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004489 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004490
Chris Masonea8efc72011-03-08 11:54:40 -05004491 /* break if the extent we found is outside the range */
4492 if (em->start >= max || extent_map_end(em) < off)
4493 break;
4494
4495 /*
4496 * get_extent may return an extent that starts before our
4497 * requested range. We have to make sure the ranges
4498 * we return to fiemap always move forward and don't
4499 * overlap, so adjust the offsets here
4500 */
4501 em_start = max(em->start, off);
4502
4503 /*
4504 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004505 * for adjusting the disk offset below. Only do this if the
4506 * extent isn't compressed since our in ram offset may be past
4507 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004508 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004509 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4510 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004511 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004512 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004513 flags = 0;
Filipe Mananaf0986312018-06-20 10:02:30 +01004514 if (em->block_start < EXTENT_MAP_LAST_BYTE)
4515 disko = em->block_start + offset_in_extent;
4516 else
4517 disko = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004518
Chris Masonea8efc72011-03-08 11:54:40 -05004519 /*
4520 * bump off for our next call to get_extent
4521 */
4522 off = extent_map_end(em);
4523 if (off >= max)
4524 end = 1;
4525
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004526 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004527 end = 1;
4528 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004529 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004530 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4531 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004532 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004533 flags |= (FIEMAP_EXTENT_DELALLOC |
4534 FIEMAP_EXTENT_UNKNOWN);
Josef Bacikdc046b12014-09-10 16:20:45 -04004535 } else if (fieinfo->fi_extents_max) {
4536 u64 bytenr = em->block_start -
4537 (em->start - em->orig_start);
Liu Bofe09e162013-09-22 12:54:23 +08004538
Liu Bofe09e162013-09-22 12:54:23 +08004539 /*
4540 * As btrfs supports shared space, this information
4541 * can be exported to userspace tools via
Josef Bacikdc046b12014-09-10 16:20:45 -04004542 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4543 * then we're just getting a count and we can skip the
4544 * lookup stuff.
Liu Bofe09e162013-09-22 12:54:23 +08004545 */
Edmund Nadolskibb739cf2017-06-28 21:56:58 -06004546 ret = btrfs_check_shared(root,
4547 btrfs_ino(BTRFS_I(inode)),
4548 bytenr);
Josef Bacikdc046b12014-09-10 16:20:45 -04004549 if (ret < 0)
Liu Bofe09e162013-09-22 12:54:23 +08004550 goto out_free;
Josef Bacikdc046b12014-09-10 16:20:45 -04004551 if (ret)
Liu Bofe09e162013-09-22 12:54:23 +08004552 flags |= FIEMAP_EXTENT_SHARED;
Josef Bacikdc046b12014-09-10 16:20:45 -04004553 ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004554 }
4555 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4556 flags |= FIEMAP_EXTENT_ENCODED;
Josef Bacik0d2b2372015-05-19 10:44:04 -04004557 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4558 flags |= FIEMAP_EXTENT_UNWRITTEN;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004559
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004560 free_extent_map(em);
4561 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004562 if ((em_start >= last) || em_len == (u64)-1 ||
4563 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004564 flags |= FIEMAP_EXTENT_LAST;
4565 end = 1;
4566 }
4567
Chris Masonec29ed52011-02-23 16:23:20 -05004568 /* now scan forward to see if this is really the last extent. */
David Sterbae3350e12017-06-23 04:09:57 +02004569 em = get_extent_skip_holes(inode, off, last_for_get_extent);
Chris Masonec29ed52011-02-23 16:23:20 -05004570 if (IS_ERR(em)) {
4571 ret = PTR_ERR(em);
4572 goto out;
4573 }
4574 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004575 flags |= FIEMAP_EXTENT_LAST;
4576 end = 1;
4577 }
Qu Wenruo47518322017-04-07 10:43:15 +08004578 ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko,
4579 em_len, flags);
Chengyu Song26e726a2015-03-24 18:12:56 -04004580 if (ret) {
4581 if (ret == 1)
4582 ret = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004583 goto out_free;
Chengyu Song26e726a2015-03-24 18:12:56 -04004584 }
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004585 }
4586out_free:
Qu Wenruo47518322017-04-07 10:43:15 +08004587 if (!ret)
Qu Wenruo848c23b2017-06-22 10:01:21 +08004588 ret = emit_last_fiemap_cache(root->fs_info, fieinfo, &cache);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004589 free_extent_map(em);
4590out:
Liu Bofe09e162013-09-22 12:54:23 +08004591 btrfs_free_path(path);
Liu Boa52f4cd2013-05-01 16:23:41 +00004592 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
David Sterbae43bbe52017-12-12 21:43:52 +01004593 &cached_state);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004594 return ret;
4595}
4596
Chris Mason727011e2010-08-06 13:21:20 -04004597static void __free_extent_buffer(struct extent_buffer *eb)
4598{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004599 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004600 kmem_cache_free(extent_buffer_cache, eb);
4601}
4602
Josef Bacika26e8c92014-03-28 17:07:27 -04004603int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004604{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004605 return (atomic_read(&eb->io_pages) ||
4606 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4607 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004608}
4609
Miao Xie897ca6e92010-10-26 20:57:29 -04004610/*
David Sterba55ac0132018-07-19 17:24:32 +02004611 * Release all pages attached to the extent buffer.
Miao Xie897ca6e92010-10-26 20:57:29 -04004612 */
David Sterba55ac0132018-07-19 17:24:32 +02004613static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
Miao Xie897ca6e92010-10-26 20:57:29 -04004614{
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004615 int i;
4616 int num_pages;
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004617 int mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
Miao Xie897ca6e92010-10-26 20:57:29 -04004618
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004619 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e92010-10-26 20:57:29 -04004620
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004621 num_pages = num_extent_pages(eb);
4622 for (i = 0; i < num_pages; i++) {
4623 struct page *page = eb->pages[i];
Miao Xie897ca6e92010-10-26 20:57:29 -04004624
Forrest Liu5d2361d2015-02-09 17:31:45 +08004625 if (!page)
4626 continue;
4627 if (mapped)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004628 spin_lock(&page->mapping->private_lock);
Forrest Liu5d2361d2015-02-09 17:31:45 +08004629 /*
4630 * We do this since we'll remove the pages after we've
4631 * removed the eb from the radix tree, so we could race
4632 * and have this page now attached to the new eb. So
4633 * only clear page_private if it's still connected to
4634 * this eb.
4635 */
4636 if (PagePrivate(page) &&
4637 page->private == (unsigned long)eb) {
4638 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4639 BUG_ON(PageDirty(page));
4640 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004641 /*
Forrest Liu5d2361d2015-02-09 17:31:45 +08004642 * We need to make sure we haven't be attached
4643 * to a new eb.
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004644 */
Forrest Liu5d2361d2015-02-09 17:31:45 +08004645 ClearPagePrivate(page);
4646 set_page_private(page, 0);
4647 /* One for the page private */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004648 put_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004649 }
Forrest Liu5d2361d2015-02-09 17:31:45 +08004650
4651 if (mapped)
4652 spin_unlock(&page->mapping->private_lock);
4653
Nicholas D Steeves01327612016-05-19 21:18:45 -04004654 /* One for when we allocated the page */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004655 put_page(page);
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004656 }
Miao Xie897ca6e92010-10-26 20:57:29 -04004657}
4658
4659/*
4660 * Helper for releasing the extent buffer.
4661 */
4662static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4663{
David Sterba55ac0132018-07-19 17:24:32 +02004664 btrfs_release_extent_buffer_pages(eb);
Miao Xie897ca6e92010-10-26 20:57:29 -04004665 __free_extent_buffer(eb);
4666}
4667
Josef Bacikf28491e2013-12-16 13:24:27 -05004668static struct extent_buffer *
4669__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
David Sterba23d79d82014-06-15 02:55:29 +02004670 unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004671{
4672 struct extent_buffer *eb = NULL;
4673
Michal Hockod1b5c562015-08-19 14:17:40 +02004674 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004675 eb->start = start;
4676 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004677 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004678 eb->bflags = 0;
4679 rwlock_init(&eb->lock);
4680 atomic_set(&eb->write_locks, 0);
4681 atomic_set(&eb->read_locks, 0);
4682 atomic_set(&eb->blocking_readers, 0);
4683 atomic_set(&eb->blocking_writers, 0);
4684 atomic_set(&eb->spinning_readers, 0);
4685 atomic_set(&eb->spinning_writers, 0);
4686 eb->lock_nested = 0;
4687 init_waitqueue_head(&eb->write_lock_wq);
4688 init_waitqueue_head(&eb->read_lock_wq);
4689
4690 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4691
4692 spin_lock_init(&eb->refs_lock);
4693 atomic_set(&eb->refs, 1);
4694 atomic_set(&eb->io_pages, 0);
4695
4696 /*
4697 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4698 */
4699 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4700 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4701 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4702
4703 return eb;
4704}
4705
4706struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4707{
David Sterbacc5e31a2018-03-01 18:20:27 +01004708 int i;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004709 struct page *p;
4710 struct extent_buffer *new;
David Sterbacc5e31a2018-03-01 18:20:27 +01004711 int num_pages = num_extent_pages(src);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004712
David Sterba3f556f72014-06-15 03:20:26 +02004713 new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004714 if (new == NULL)
4715 return NULL;
4716
4717 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004718 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004719 if (!p) {
4720 btrfs_release_extent_buffer(new);
4721 return NULL;
4722 }
4723 attach_extent_buffer_page(new, p);
4724 WARN_ON(PageDirty(p));
4725 SetPageUptodate(p);
4726 new->pages[i] = p;
David Sterbafba1acf2016-11-08 17:56:24 +01004727 copy_page(page_address(p), page_address(src->pages[i]));
Josef Bacikdb7f3432013-08-07 14:54:37 -04004728 }
4729
Josef Bacikdb7f3432013-08-07 14:54:37 -04004730 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004731 set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004732
4733 return new;
4734}
4735
Omar Sandoval0f331222015-09-29 20:50:31 -07004736struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4737 u64 start, unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004738{
4739 struct extent_buffer *eb;
David Sterbacc5e31a2018-03-01 18:20:27 +01004740 int num_pages;
4741 int i;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004742
David Sterba3f556f72014-06-15 03:20:26 +02004743 eb = __alloc_extent_buffer(fs_info, start, len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004744 if (!eb)
4745 return NULL;
4746
David Sterba65ad0102018-06-29 10:56:49 +02004747 num_pages = num_extent_pages(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004748 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004749 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004750 if (!eb->pages[i])
4751 goto err;
4752 }
4753 set_extent_buffer_uptodate(eb);
4754 btrfs_set_header_nritems(eb, 0);
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004755 set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004756
4757 return eb;
4758err:
4759 for (; i > 0; i--)
4760 __free_page(eb->pages[i - 1]);
4761 __free_extent_buffer(eb);
4762 return NULL;
4763}
4764
Omar Sandoval0f331222015-09-29 20:50:31 -07004765struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
Jeff Mahoneyda170662016-06-15 09:22:56 -04004766 u64 start)
Omar Sandoval0f331222015-09-29 20:50:31 -07004767{
Jeff Mahoneyda170662016-06-15 09:22:56 -04004768 return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
Omar Sandoval0f331222015-09-29 20:50:31 -07004769}
4770
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004771static void check_buffer_tree_ref(struct extent_buffer *eb)
4772{
Chris Mason242e18c2013-01-29 17:49:37 -05004773 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004774 /* the ref bit is tricky. We have to make sure it is set
4775 * if we have the buffer dirty. Otherwise the
4776 * code to free a buffer can end up dropping a dirty
4777 * page
4778 *
4779 * Once the ref bit is set, it won't go away while the
4780 * buffer is dirty or in writeback, and it also won't
4781 * go away while we have the reference count on the
4782 * eb bumped.
4783 *
4784 * We can't just set the ref bit without bumping the
4785 * ref on the eb because free_extent_buffer might
4786 * see the ref bit and try to clear it. If this happens
4787 * free_extent_buffer might end up dropping our original
4788 * ref by mistake and freeing the page before we are able
4789 * to add one more ref.
4790 *
4791 * So bump the ref count first, then set the bit. If someone
4792 * beat us to it, drop the ref we added.
4793 */
Chris Mason242e18c2013-01-29 17:49:37 -05004794 refs = atomic_read(&eb->refs);
4795 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4796 return;
4797
Josef Bacik594831c2012-07-20 16:11:08 -04004798 spin_lock(&eb->refs_lock);
4799 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004800 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004801 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004802}
4803
Mel Gorman2457aec2014-06-04 16:10:31 -07004804static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4805 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04004806{
David Sterbacc5e31a2018-03-01 18:20:27 +01004807 int num_pages, i;
Josef Bacik5df42352012-03-15 18:24:42 -04004808
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004809 check_buffer_tree_ref(eb);
4810
David Sterba65ad0102018-06-29 10:56:49 +02004811 num_pages = num_extent_pages(eb);
Josef Bacik5df42352012-03-15 18:24:42 -04004812 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02004813 struct page *p = eb->pages[i];
4814
Mel Gorman2457aec2014-06-04 16:10:31 -07004815 if (p != accessed)
4816 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004817 }
4818}
4819
Josef Bacikf28491e2013-12-16 13:24:27 -05004820struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4821 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004822{
4823 struct extent_buffer *eb;
4824
4825 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05004826 eb = radix_tree_lookup(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004827 start >> PAGE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004828 if (eb && atomic_inc_not_zero(&eb->refs)) {
4829 rcu_read_unlock();
Filipe Manana062c19e2015-04-23 11:28:48 +01004830 /*
4831 * Lock our eb's refs_lock to avoid races with
4832 * free_extent_buffer. When we get our eb it might be flagged
4833 * with EXTENT_BUFFER_STALE and another task running
4834 * free_extent_buffer might have seen that flag set,
4835 * eb->refs == 2, that the buffer isn't under IO (dirty and
4836 * writeback flags not set) and it's still in the tree (flag
4837 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
4838 * of decrementing the extent buffer's reference count twice.
4839 * So here we could race and increment the eb's reference count,
4840 * clear its stale flag, mark it as dirty and drop our reference
4841 * before the other task finishes executing free_extent_buffer,
4842 * which would later result in an attempt to free an extent
4843 * buffer that is dirty.
4844 */
4845 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
4846 spin_lock(&eb->refs_lock);
4847 spin_unlock(&eb->refs_lock);
4848 }
Mel Gorman2457aec2014-06-04 16:10:31 -07004849 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004850 return eb;
4851 }
4852 rcu_read_unlock();
4853
4854 return NULL;
4855}
4856
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004857#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4858struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
Jeff Mahoneyda170662016-06-15 09:22:56 -04004859 u64 start)
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004860{
4861 struct extent_buffer *eb, *exists = NULL;
4862 int ret;
4863
4864 eb = find_extent_buffer(fs_info, start);
4865 if (eb)
4866 return eb;
Jeff Mahoneyda170662016-06-15 09:22:56 -04004867 eb = alloc_dummy_extent_buffer(fs_info, start);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004868 if (!eb)
4869 return NULL;
4870 eb->fs_info = fs_info;
4871again:
David Sterbae1860a72016-05-09 14:11:38 +02004872 ret = radix_tree_preload(GFP_NOFS);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004873 if (ret)
4874 goto free_eb;
4875 spin_lock(&fs_info->buffer_lock);
4876 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004877 start >> PAGE_SHIFT, eb);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004878 spin_unlock(&fs_info->buffer_lock);
4879 radix_tree_preload_end();
4880 if (ret == -EEXIST) {
4881 exists = find_extent_buffer(fs_info, start);
4882 if (exists)
4883 goto free_eb;
4884 else
4885 goto again;
4886 }
4887 check_buffer_tree_ref(eb);
4888 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4889
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004890 return eb;
4891free_eb:
4892 btrfs_release_extent_buffer(eb);
4893 return exists;
4894}
4895#endif
4896
Josef Bacikf28491e2013-12-16 13:24:27 -05004897struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02004898 u64 start)
Chris Masond1310b22008-01-24 16:13:08 -05004899{
Jeff Mahoneyda170662016-06-15 09:22:56 -04004900 unsigned long len = fs_info->nodesize;
David Sterbacc5e31a2018-03-01 18:20:27 +01004901 int num_pages;
4902 int i;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004903 unsigned long index = start >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05004904 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004905 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004906 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05004907 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05004908 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004909 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004910
Jeff Mahoneyda170662016-06-15 09:22:56 -04004911 if (!IS_ALIGNED(start, fs_info->sectorsize)) {
Liu Boc871b0f2016-06-06 12:01:23 -07004912 btrfs_err(fs_info, "bad tree block start %llu", start);
4913 return ERR_PTR(-EINVAL);
4914 }
4915
Josef Bacikf28491e2013-12-16 13:24:27 -05004916 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004917 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04004918 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004919
David Sterba23d79d82014-06-15 02:55:29 +02004920 eb = __alloc_extent_buffer(fs_info, start, len);
Peter2b114d12008-04-01 11:21:40 -04004921 if (!eb)
Liu Boc871b0f2016-06-06 12:01:23 -07004922 return ERR_PTR(-ENOMEM);
Chris Masond1310b22008-01-24 16:13:08 -05004923
David Sterba65ad0102018-06-29 10:56:49 +02004924 num_pages = num_extent_pages(eb);
Chris Mason727011e2010-08-06 13:21:20 -04004925 for (i = 0; i < num_pages; i++, index++) {
Michal Hockod1b5c562015-08-19 14:17:40 +02004926 p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
Liu Boc871b0f2016-06-06 12:01:23 -07004927 if (!p) {
4928 exists = ERR_PTR(-ENOMEM);
Chris Mason6af118ce2008-07-22 11:18:07 -04004929 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07004930 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004931
4932 spin_lock(&mapping->private_lock);
4933 if (PagePrivate(p)) {
4934 /*
4935 * We could have already allocated an eb for this page
4936 * and attached one so lets see if we can get a ref on
4937 * the existing eb, and if we can we know it's good and
4938 * we can just return that one, else we know we can just
4939 * overwrite page->private.
4940 */
4941 exists = (struct extent_buffer *)p->private;
4942 if (atomic_inc_not_zero(&exists->refs)) {
4943 spin_unlock(&mapping->private_lock);
4944 unlock_page(p);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004945 put_page(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07004946 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004947 goto free_eb;
4948 }
Omar Sandoval5ca64f42015-02-24 02:47:05 -08004949 exists = NULL;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004950
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004951 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004952 * Do this so attach doesn't complain and we need to
4953 * drop the ref the old guy had.
4954 */
4955 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004956 WARN_ON(PageDirty(p));
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004957 put_page(p);
Chris Masond1310b22008-01-24 16:13:08 -05004958 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004959 attach_extent_buffer_page(eb, p);
4960 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004961 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04004962 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004963 if (!PageUptodate(p))
4964 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004965
4966 /*
Nikolay Borisovb16d0112018-07-04 10:24:52 +03004967 * We can't unlock the pages just yet since the extent buffer
4968 * hasn't been properly inserted in the radix tree, this
4969 * opens a race with btree_releasepage which can free a page
4970 * while we are still filling in all pages for the buffer and
4971 * we could crash.
Chris Masoneb14ab82011-02-10 12:35:00 -05004972 */
Chris Masond1310b22008-01-24 16:13:08 -05004973 }
4974 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004975 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004976again:
David Sterbae1860a72016-05-09 14:11:38 +02004977 ret = radix_tree_preload(GFP_NOFS);
Liu Boc871b0f2016-06-06 12:01:23 -07004978 if (ret) {
4979 exists = ERR_PTR(ret);
Miao Xie19fe0a82010-10-26 20:57:29 -04004980 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07004981 }
Miao Xie19fe0a82010-10-26 20:57:29 -04004982
Josef Bacikf28491e2013-12-16 13:24:27 -05004983 spin_lock(&fs_info->buffer_lock);
4984 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004985 start >> PAGE_SHIFT, eb);
Josef Bacikf28491e2013-12-16 13:24:27 -05004986 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004987 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04004988 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004989 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004990 if (exists)
4991 goto free_eb;
4992 else
Josef Bacik115391d2012-03-09 09:51:43 -05004993 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04004994 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004995 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004996 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004997 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05004998
4999 /*
Nikolay Borisovb16d0112018-07-04 10:24:52 +03005000 * Now it's safe to unlock the pages because any calls to
5001 * btree_releasepage will correctly detect that a page belongs to a
5002 * live buffer and won't free them prematurely.
Chris Masoneb14ab82011-02-10 12:35:00 -05005003 */
Nikolay Borisov28187ae2018-07-04 10:24:51 +03005004 for (i = 0; i < num_pages; i++)
5005 unlock_page(eb->pages[i]);
Chris Masond1310b22008-01-24 16:13:08 -05005006 return eb;
5007
Chris Mason6af118ce2008-07-22 11:18:07 -04005008free_eb:
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005009 WARN_ON(!atomic_dec_and_test(&eb->refs));
Chris Mason727011e2010-08-06 13:21:20 -04005010 for (i = 0; i < num_pages; i++) {
5011 if (eb->pages[i])
5012 unlock_page(eb->pages[i]);
5013 }
Chris Masoneb14ab82011-02-10 12:35:00 -05005014
Miao Xie897ca6e92010-10-26 20:57:29 -04005015 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005016 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05005017}
Chris Masond1310b22008-01-24 16:13:08 -05005018
Josef Bacik3083ee22012-03-09 16:01:49 -05005019static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
5020{
5021 struct extent_buffer *eb =
5022 container_of(head, struct extent_buffer, rcu_head);
5023
5024 __free_extent_buffer(eb);
5025}
5026
David Sterbaf7a52a42013-04-26 14:56:29 +00005027static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05005028{
Nikolay Borisov07e21c42018-06-27 16:38:23 +03005029 lockdep_assert_held(&eb->refs_lock);
5030
Josef Bacik3083ee22012-03-09 16:01:49 -05005031 WARN_ON(atomic_read(&eb->refs) == 0);
5032 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05005033 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005034 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05005035
Jan Schmidt815a51c2012-05-16 17:00:02 +02005036 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05005037
Josef Bacikf28491e2013-12-16 13:24:27 -05005038 spin_lock(&fs_info->buffer_lock);
5039 radix_tree_delete(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005040 eb->start >> PAGE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05005041 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005042 } else {
5043 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02005044 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005045
5046 /* Should be safe to release our pages at this point */
David Sterba55ac0132018-07-19 17:24:32 +02005047 btrfs_release_extent_buffer_pages(eb);
Josef Bacikbcb7e442015-03-16 17:38:02 -04005048#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
Nikolay Borisovb0132a32018-06-27 16:38:24 +03005049 if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
Josef Bacikbcb7e442015-03-16 17:38:02 -04005050 __free_extent_buffer(eb);
5051 return 1;
5052 }
5053#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05005054 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04005055 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05005056 }
5057 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04005058
5059 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05005060}
5061
Chris Masond1310b22008-01-24 16:13:08 -05005062void free_extent_buffer(struct extent_buffer *eb)
5063{
Chris Mason242e18c2013-01-29 17:49:37 -05005064 int refs;
5065 int old;
Chris Masond1310b22008-01-24 16:13:08 -05005066 if (!eb)
5067 return;
5068
Chris Mason242e18c2013-01-29 17:49:37 -05005069 while (1) {
5070 refs = atomic_read(&eb->refs);
Nikolay Borisov46cc7752018-10-15 17:04:01 +03005071 if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
5072 || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
5073 refs == 1))
Chris Mason242e18c2013-01-29 17:49:37 -05005074 break;
5075 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5076 if (old == refs)
5077 return;
5078 }
5079
Josef Bacik3083ee22012-03-09 16:01:49 -05005080 spin_lock(&eb->refs_lock);
5081 if (atomic_read(&eb->refs) == 2 &&
5082 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005083 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005084 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5085 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05005086
Josef Bacik3083ee22012-03-09 16:01:49 -05005087 /*
5088 * I know this is terrible, but it's temporary until we stop tracking
5089 * the uptodate bits and such for the extent buffers.
5090 */
David Sterbaf7a52a42013-04-26 14:56:29 +00005091 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005092}
Chris Masond1310b22008-01-24 16:13:08 -05005093
Josef Bacik3083ee22012-03-09 16:01:49 -05005094void free_extent_buffer_stale(struct extent_buffer *eb)
5095{
5096 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05005097 return;
5098
Josef Bacik3083ee22012-03-09 16:01:49 -05005099 spin_lock(&eb->refs_lock);
5100 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5101
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005102 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005103 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5104 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00005105 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005106}
5107
Chris Mason1d4284b2012-03-28 20:31:37 -04005108void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005109{
David Sterbacc5e31a2018-03-01 18:20:27 +01005110 int i;
5111 int num_pages;
Chris Masond1310b22008-01-24 16:13:08 -05005112 struct page *page;
5113
David Sterba65ad0102018-06-29 10:56:49 +02005114 num_pages = num_extent_pages(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005115
5116 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005117 page = eb->pages[i];
Chris Masonb9473432009-03-13 11:00:37 -04005118 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05005119 continue;
5120
Chris Masona61e6f22008-07-22 11:18:08 -04005121 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05005122 WARN_ON(!PagePrivate(page));
5123
Chris Masond1310b22008-01-24 16:13:08 -05005124 clear_page_dirty_for_io(page);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07005125 xa_lock_irq(&page->mapping->i_pages);
Matthew Wilcox0a943c62017-12-04 10:37:22 -05005126 if (!PageDirty(page))
5127 __xa_clear_mark(&page->mapping->i_pages,
5128 page_index(page), PAGECACHE_TAG_DIRTY);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07005129 xa_unlock_irq(&page->mapping->i_pages);
Chris Masonbf0da8c2011-11-04 12:29:37 -04005130 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04005131 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05005132 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005133 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05005134}
Chris Masond1310b22008-01-24 16:13:08 -05005135
Liu Boabb57ef2018-09-14 01:44:42 +08005136bool set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005137{
David Sterbacc5e31a2018-03-01 18:20:27 +01005138 int i;
5139 int num_pages;
Liu Boabb57ef2018-09-14 01:44:42 +08005140 bool was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005141
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005142 check_buffer_tree_ref(eb);
5143
Chris Masonb9473432009-03-13 11:00:37 -04005144 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005145
David Sterba65ad0102018-06-29 10:56:49 +02005146 num_pages = num_extent_pages(eb);
Josef Bacik3083ee22012-03-09 16:01:49 -05005147 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005148 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5149
Liu Boabb57ef2018-09-14 01:44:42 +08005150 if (!was_dirty)
5151 for (i = 0; i < num_pages; i++)
5152 set_page_dirty(eb->pages[i]);
Liu Bo51995c32018-09-14 01:46:08 +08005153
5154#ifdef CONFIG_BTRFS_DEBUG
5155 for (i = 0; i < num_pages; i++)
5156 ASSERT(PageDirty(eb->pages[i]));
5157#endif
5158
Chris Masonb9473432009-03-13 11:00:37 -04005159 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005160}
Chris Masond1310b22008-01-24 16:13:08 -05005161
David Sterba69ba3922015-12-03 13:08:59 +01005162void clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04005163{
David Sterbacc5e31a2018-03-01 18:20:27 +01005164 int i;
Chris Mason1259ab72008-05-12 13:39:03 -04005165 struct page *page;
David Sterbacc5e31a2018-03-01 18:20:27 +01005166 int num_pages;
Chris Mason1259ab72008-05-12 13:39:03 -04005167
Chris Masonb4ce94d2009-02-04 09:25:08 -05005168 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02005169 num_pages = num_extent_pages(eb);
Chris Mason1259ab72008-05-12 13:39:03 -04005170 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005171 page = eb->pages[i];
Chris Mason33958dc2008-07-30 10:29:12 -04005172 if (page)
5173 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04005174 }
Chris Mason1259ab72008-05-12 13:39:03 -04005175}
5176
David Sterba09c25a82015-12-03 13:08:59 +01005177void set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005178{
David Sterbacc5e31a2018-03-01 18:20:27 +01005179 int i;
Chris Masond1310b22008-01-24 16:13:08 -05005180 struct page *page;
David Sterbacc5e31a2018-03-01 18:20:27 +01005181 int num_pages;
Chris Masond1310b22008-01-24 16:13:08 -05005182
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005183 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02005184 num_pages = num_extent_pages(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005185 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005186 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005187 SetPageUptodate(page);
5188 }
Chris Masond1310b22008-01-24 16:13:08 -05005189}
Chris Masond1310b22008-01-24 16:13:08 -05005190
Chris Masond1310b22008-01-24 16:13:08 -05005191int read_extent_buffer_pages(struct extent_io_tree *tree,
David Sterba6af49db2017-06-23 04:09:57 +02005192 struct extent_buffer *eb, int wait, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05005193{
David Sterbacc5e31a2018-03-01 18:20:27 +01005194 int i;
Chris Masond1310b22008-01-24 16:13:08 -05005195 struct page *page;
5196 int err;
5197 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04005198 int locked_pages = 0;
5199 int all_uptodate = 1;
David Sterbacc5e31a2018-03-01 18:20:27 +01005200 int num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04005201 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005202 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04005203 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005204
Chris Masonb4ce94d2009-02-04 09:25:08 -05005205 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05005206 return 0;
5207
David Sterba65ad0102018-06-29 10:56:49 +02005208 num_pages = num_extent_pages(eb);
Josef Bacik8436ea912016-09-02 15:40:03 -04005209 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005210 page = eb->pages[i];
Arne Jansenbb82ab82011-06-10 14:06:53 +02005211 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04005212 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04005213 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05005214 } else {
5215 lock_page(page);
5216 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005217 locked_pages++;
Liu Bo2571e732016-08-03 12:33:01 -07005218 }
5219 /*
5220 * We need to firstly lock all pages to make sure that
5221 * the uptodate bit of our pages won't be affected by
5222 * clear_extent_buffer_uptodate().
5223 */
Josef Bacik8436ea912016-09-02 15:40:03 -04005224 for (i = 0; i < num_pages; i++) {
Liu Bo2571e732016-08-03 12:33:01 -07005225 page = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04005226 if (!PageUptodate(page)) {
5227 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04005228 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04005229 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005230 }
Liu Bo2571e732016-08-03 12:33:01 -07005231
Chris Masonce9adaa2008-04-09 16:28:12 -04005232 if (all_uptodate) {
Josef Bacik8436ea912016-09-02 15:40:03 -04005233 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04005234 goto unlock_exit;
5235 }
5236
Filipe Manana656f30d2014-09-26 12:25:56 +01005237 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04005238 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005239 atomic_set(&eb->io_pages, num_reads);
Josef Bacik8436ea912016-09-02 15:40:03 -04005240 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005241 page = eb->pages[i];
Liu Bobaf863b2016-07-11 10:39:07 -07005242
Chris Masonce9adaa2008-04-09 16:28:12 -04005243 if (!PageUptodate(page)) {
Liu Bobaf863b2016-07-11 10:39:07 -07005244 if (ret) {
5245 atomic_dec(&eb->io_pages);
5246 unlock_page(page);
5247 continue;
5248 }
5249
Chris Masonf1885912008-04-09 16:28:12 -04005250 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05005251 err = __extent_read_full_page(tree, page,
David Sterba6af49db2017-06-23 04:09:57 +02005252 btree_get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005253 mirror_num, &bio_flags,
Mike Christie1f7ad752016-06-05 14:31:51 -05005254 REQ_META);
Liu Bobaf863b2016-07-11 10:39:07 -07005255 if (err) {
Chris Masond1310b22008-01-24 16:13:08 -05005256 ret = err;
Liu Bobaf863b2016-07-11 10:39:07 -07005257 /*
5258 * We use &bio in above __extent_read_full_page,
5259 * so we ensure that if it returns error, the
5260 * current page fails to add itself to bio and
5261 * it's been unlocked.
5262 *
5263 * We must dec io_pages by ourselves.
5264 */
5265 atomic_dec(&eb->io_pages);
5266 }
Chris Masond1310b22008-01-24 16:13:08 -05005267 } else {
5268 unlock_page(page);
5269 }
5270 }
5271
Jeff Mahoney355808c2011-10-03 23:23:14 -04005272 if (bio) {
Mike Christie1f7ad752016-06-05 14:31:51 -05005273 err = submit_one_bio(bio, mirror_num, bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005274 if (err)
5275 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005276 }
Chris Masona86c12c2008-02-07 10:50:54 -05005277
Arne Jansenbb82ab82011-06-10 14:06:53 +02005278 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005279 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005280
Josef Bacik8436ea912016-09-02 15:40:03 -04005281 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005282 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005283 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005284 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005285 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005286 }
Chris Masond3977122009-01-05 21:25:51 -05005287
Chris Masond1310b22008-01-24 16:13:08 -05005288 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005289
5290unlock_exit:
Chris Masond3977122009-01-05 21:25:51 -05005291 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04005292 locked_pages--;
Josef Bacik8436ea912016-09-02 15:40:03 -04005293 page = eb->pages[locked_pages];
5294 unlock_page(page);
Chris Masonce9adaa2008-04-09 16:28:12 -04005295 }
5296 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005297}
Chris Masond1310b22008-01-24 16:13:08 -05005298
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005299void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
5300 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005301{
5302 size_t cur;
5303 size_t offset;
5304 struct page *page;
5305 char *kaddr;
5306 char *dst = (char *)dstv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005307 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005308 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005309
Liu Bof716abd2017-08-09 11:10:16 -06005310 if (start + len > eb->len) {
5311 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5312 eb->start, eb->len, start, len);
5313 memset(dst, 0, len);
5314 return;
5315 }
Chris Masond1310b22008-01-24 16:13:08 -05005316
Johannes Thumshirn70730172018-12-05 15:23:03 +01005317 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005318
Chris Masond3977122009-01-05 21:25:51 -05005319 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005320 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005321
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005322 cur = min(len, (PAGE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005323 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005324 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005325
5326 dst += cur;
5327 len -= cur;
5328 offset = 0;
5329 i++;
5330 }
5331}
Chris Masond1310b22008-01-24 16:13:08 -05005332
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005333int read_extent_buffer_to_user(const struct extent_buffer *eb,
5334 void __user *dstv,
5335 unsigned long start, unsigned long len)
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005336{
5337 size_t cur;
5338 size_t offset;
5339 struct page *page;
5340 char *kaddr;
5341 char __user *dst = (char __user *)dstv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005342 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005343 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005344 int ret = 0;
5345
5346 WARN_ON(start > eb->len);
5347 WARN_ON(start + len > eb->start + eb->len);
5348
Johannes Thumshirn70730172018-12-05 15:23:03 +01005349 offset = offset_in_page(start_offset + start);
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005350
5351 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005352 page = eb->pages[i];
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005353
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005354 cur = min(len, (PAGE_SIZE - offset));
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005355 kaddr = page_address(page);
5356 if (copy_to_user(dst, kaddr + offset, cur)) {
5357 ret = -EFAULT;
5358 break;
5359 }
5360
5361 dst += cur;
5362 len -= cur;
5363 offset = 0;
5364 i++;
5365 }
5366
5367 return ret;
5368}
5369
Liu Bo415b35a2016-06-17 19:16:21 -07005370/*
5371 * return 0 if the item is found within a page.
5372 * return 1 if the item spans two pages.
5373 * return -EINVAL otherwise.
5374 */
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005375int map_private_extent_buffer(const struct extent_buffer *eb,
5376 unsigned long start, unsigned long min_len,
5377 char **map, unsigned long *map_start,
5378 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005379{
Johannes Thumshirncc2c39d2018-11-28 09:54:54 +01005380 size_t offset;
Chris Masond1310b22008-01-24 16:13:08 -05005381 char *kaddr;
5382 struct page *p;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005383 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005384 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005385 unsigned long end_i = (start_offset + start + min_len - 1) >>
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005386 PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005387
Liu Bof716abd2017-08-09 11:10:16 -06005388 if (start + min_len > eb->len) {
5389 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5390 eb->start, eb->len, start, min_len);
5391 return -EINVAL;
5392 }
5393
Chris Masond1310b22008-01-24 16:13:08 -05005394 if (i != end_i)
Liu Bo415b35a2016-06-17 19:16:21 -07005395 return 1;
Chris Masond1310b22008-01-24 16:13:08 -05005396
5397 if (i == 0) {
5398 offset = start_offset;
5399 *map_start = 0;
5400 } else {
5401 offset = 0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005402 *map_start = ((u64)i << PAGE_SHIFT) - start_offset;
Chris Masond1310b22008-01-24 16:13:08 -05005403 }
Chris Masond3977122009-01-05 21:25:51 -05005404
David Sterbafb85fc92014-07-31 01:03:53 +02005405 p = eb->pages[i];
Chris Masona6591712011-07-19 12:04:14 -04005406 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005407 *map = kaddr + offset;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005408 *map_len = PAGE_SIZE - offset;
Chris Masond1310b22008-01-24 16:13:08 -05005409 return 0;
5410}
Chris Masond1310b22008-01-24 16:13:08 -05005411
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005412int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
5413 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005414{
5415 size_t cur;
5416 size_t offset;
5417 struct page *page;
5418 char *kaddr;
5419 char *ptr = (char *)ptrv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005420 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005421 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005422 int ret = 0;
5423
5424 WARN_ON(start > eb->len);
5425 WARN_ON(start + len > eb->start + eb->len);
5426
Johannes Thumshirn70730172018-12-05 15:23:03 +01005427 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005428
Chris Masond3977122009-01-05 21:25:51 -05005429 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005430 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005431
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005432 cur = min(len, (PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005433
Chris Masona6591712011-07-19 12:04:14 -04005434 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005435 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005436 if (ret)
5437 break;
5438
5439 ptr += cur;
5440 len -= cur;
5441 offset = 0;
5442 i++;
5443 }
5444 return ret;
5445}
Chris Masond1310b22008-01-24 16:13:08 -05005446
David Sterbaf157bf72016-11-09 17:43:38 +01005447void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb,
5448 const void *srcv)
5449{
5450 char *kaddr;
5451
5452 WARN_ON(!PageUptodate(eb->pages[0]));
5453 kaddr = page_address(eb->pages[0]);
5454 memcpy(kaddr + offsetof(struct btrfs_header, chunk_tree_uuid), srcv,
5455 BTRFS_FSID_SIZE);
5456}
5457
5458void write_extent_buffer_fsid(struct extent_buffer *eb, const void *srcv)
5459{
5460 char *kaddr;
5461
5462 WARN_ON(!PageUptodate(eb->pages[0]));
5463 kaddr = page_address(eb->pages[0]);
5464 memcpy(kaddr + offsetof(struct btrfs_header, fsid), srcv,
5465 BTRFS_FSID_SIZE);
5466}
5467
Chris Masond1310b22008-01-24 16:13:08 -05005468void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5469 unsigned long start, unsigned long len)
5470{
5471 size_t cur;
5472 size_t offset;
5473 struct page *page;
5474 char *kaddr;
5475 char *src = (char *)srcv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005476 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005477 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005478
5479 WARN_ON(start > eb->len);
5480 WARN_ON(start + len > eb->start + eb->len);
5481
Johannes Thumshirn70730172018-12-05 15:23:03 +01005482 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005483
Chris Masond3977122009-01-05 21:25:51 -05005484 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005485 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005486 WARN_ON(!PageUptodate(page));
5487
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005488 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005489 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005490 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005491
5492 src += cur;
5493 len -= cur;
5494 offset = 0;
5495 i++;
5496 }
5497}
Chris Masond1310b22008-01-24 16:13:08 -05005498
David Sterbab159fa22016-11-08 18:09:03 +01005499void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start,
5500 unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005501{
5502 size_t cur;
5503 size_t offset;
5504 struct page *page;
5505 char *kaddr;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005506 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005507 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005508
5509 WARN_ON(start > eb->len);
5510 WARN_ON(start + len > eb->start + eb->len);
5511
Johannes Thumshirn70730172018-12-05 15:23:03 +01005512 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005513
Chris Masond3977122009-01-05 21:25:51 -05005514 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005515 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005516 WARN_ON(!PageUptodate(page));
5517
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005518 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005519 kaddr = page_address(page);
David Sterbab159fa22016-11-08 18:09:03 +01005520 memset(kaddr + offset, 0, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005521
5522 len -= cur;
5523 offset = 0;
5524 i++;
5525 }
5526}
Chris Masond1310b22008-01-24 16:13:08 -05005527
David Sterba58e80122016-11-08 18:30:31 +01005528void copy_extent_buffer_full(struct extent_buffer *dst,
5529 struct extent_buffer *src)
5530{
5531 int i;
David Sterbacc5e31a2018-03-01 18:20:27 +01005532 int num_pages;
David Sterba58e80122016-11-08 18:30:31 +01005533
5534 ASSERT(dst->len == src->len);
5535
David Sterba65ad0102018-06-29 10:56:49 +02005536 num_pages = num_extent_pages(dst);
David Sterba58e80122016-11-08 18:30:31 +01005537 for (i = 0; i < num_pages; i++)
5538 copy_page(page_address(dst->pages[i]),
5539 page_address(src->pages[i]));
5540}
5541
Chris Masond1310b22008-01-24 16:13:08 -05005542void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5543 unsigned long dst_offset, unsigned long src_offset,
5544 unsigned long len)
5545{
5546 u64 dst_len = dst->len;
5547 size_t cur;
5548 size_t offset;
5549 struct page *page;
5550 char *kaddr;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005551 size_t start_offset = offset_in_page(dst->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005552 unsigned long i = (start_offset + dst_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005553
5554 WARN_ON(src->len != dst_len);
5555
Johannes Thumshirn70730172018-12-05 15:23:03 +01005556 offset = offset_in_page(start_offset + dst_offset);
Chris Masond1310b22008-01-24 16:13:08 -05005557
Chris Masond3977122009-01-05 21:25:51 -05005558 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005559 page = dst->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005560 WARN_ON(!PageUptodate(page));
5561
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005562 cur = min(len, (unsigned long)(PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005563
Chris Masona6591712011-07-19 12:04:14 -04005564 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005565 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005566
5567 src_offset += cur;
5568 len -= cur;
5569 offset = 0;
5570 i++;
5571 }
5572}
Chris Masond1310b22008-01-24 16:13:08 -05005573
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005574/*
5575 * eb_bitmap_offset() - calculate the page and offset of the byte containing the
5576 * given bit number
5577 * @eb: the extent buffer
5578 * @start: offset of the bitmap item in the extent buffer
5579 * @nr: bit number
5580 * @page_index: return index of the page in the extent buffer that contains the
5581 * given bit number
5582 * @page_offset: return offset into the page given by page_index
5583 *
5584 * This helper hides the ugliness of finding the byte in an extent buffer which
5585 * contains a given bit.
5586 */
5587static inline void eb_bitmap_offset(struct extent_buffer *eb,
5588 unsigned long start, unsigned long nr,
5589 unsigned long *page_index,
5590 size_t *page_offset)
5591{
Johannes Thumshirn70730172018-12-05 15:23:03 +01005592 size_t start_offset = offset_in_page(eb->start);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005593 size_t byte_offset = BIT_BYTE(nr);
5594 size_t offset;
5595
5596 /*
5597 * The byte we want is the offset of the extent buffer + the offset of
5598 * the bitmap item in the extent buffer + the offset of the byte in the
5599 * bitmap item.
5600 */
5601 offset = start_offset + start + byte_offset;
5602
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005603 *page_index = offset >> PAGE_SHIFT;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005604 *page_offset = offset_in_page(offset);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005605}
5606
5607/**
5608 * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
5609 * @eb: the extent buffer
5610 * @start: offset of the bitmap item in the extent buffer
5611 * @nr: bit number to test
5612 */
5613int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start,
5614 unsigned long nr)
5615{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005616 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005617 struct page *page;
5618 unsigned long i;
5619 size_t offset;
5620
5621 eb_bitmap_offset(eb, start, nr, &i, &offset);
5622 page = eb->pages[i];
5623 WARN_ON(!PageUptodate(page));
5624 kaddr = page_address(page);
5625 return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
5626}
5627
5628/**
5629 * extent_buffer_bitmap_set - set an area of a bitmap
5630 * @eb: the extent buffer
5631 * @start: offset of the bitmap item in the extent buffer
5632 * @pos: bit number of the first bit
5633 * @len: number of bits to set
5634 */
5635void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start,
5636 unsigned long pos, unsigned long len)
5637{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005638 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005639 struct page *page;
5640 unsigned long i;
5641 size_t offset;
5642 const unsigned int size = pos + len;
5643 int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005644 u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005645
5646 eb_bitmap_offset(eb, start, pos, &i, &offset);
5647 page = eb->pages[i];
5648 WARN_ON(!PageUptodate(page));
5649 kaddr = page_address(page);
5650
5651 while (len >= bits_to_set) {
5652 kaddr[offset] |= mask_to_set;
5653 len -= bits_to_set;
5654 bits_to_set = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005655 mask_to_set = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005656 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005657 offset = 0;
5658 page = eb->pages[++i];
5659 WARN_ON(!PageUptodate(page));
5660 kaddr = page_address(page);
5661 }
5662 }
5663 if (len) {
5664 mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
5665 kaddr[offset] |= mask_to_set;
5666 }
5667}
5668
5669
5670/**
5671 * extent_buffer_bitmap_clear - clear an area of a bitmap
5672 * @eb: the extent buffer
5673 * @start: offset of the bitmap item in the extent buffer
5674 * @pos: bit number of the first bit
5675 * @len: number of bits to clear
5676 */
5677void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start,
5678 unsigned long pos, unsigned long len)
5679{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005680 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005681 struct page *page;
5682 unsigned long i;
5683 size_t offset;
5684 const unsigned int size = pos + len;
5685 int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005686 u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005687
5688 eb_bitmap_offset(eb, start, pos, &i, &offset);
5689 page = eb->pages[i];
5690 WARN_ON(!PageUptodate(page));
5691 kaddr = page_address(page);
5692
5693 while (len >= bits_to_clear) {
5694 kaddr[offset] &= ~mask_to_clear;
5695 len -= bits_to_clear;
5696 bits_to_clear = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005697 mask_to_clear = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005698 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005699 offset = 0;
5700 page = eb->pages[++i];
5701 WARN_ON(!PageUptodate(page));
5702 kaddr = page_address(page);
5703 }
5704 }
5705 if (len) {
5706 mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
5707 kaddr[offset] &= ~mask_to_clear;
5708 }
5709}
5710
Sergei Trofimovich33872062011-04-11 21:52:52 +00005711static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5712{
5713 unsigned long distance = (src > dst) ? src - dst : dst - src;
5714 return distance < len;
5715}
5716
Chris Masond1310b22008-01-24 16:13:08 -05005717static void copy_pages(struct page *dst_page, struct page *src_page,
5718 unsigned long dst_off, unsigned long src_off,
5719 unsigned long len)
5720{
Chris Masona6591712011-07-19 12:04:14 -04005721 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005722 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005723 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005724
Sergei Trofimovich33872062011-04-11 21:52:52 +00005725 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005726 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005727 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005728 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005729 if (areas_overlap(src_off, dst_off, len))
5730 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005731 }
Chris Masond1310b22008-01-24 16:13:08 -05005732
Chris Mason727011e2010-08-06 13:21:20 -04005733 if (must_memmove)
5734 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5735 else
5736 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005737}
5738
5739void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5740 unsigned long src_offset, unsigned long len)
5741{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005742 struct btrfs_fs_info *fs_info = dst->fs_info;
Chris Masond1310b22008-01-24 16:13:08 -05005743 size_t cur;
5744 size_t dst_off_in_page;
5745 size_t src_off_in_page;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005746 size_t start_offset = offset_in_page(dst->start);
Chris Masond1310b22008-01-24 16:13:08 -05005747 unsigned long dst_i;
5748 unsigned long src_i;
5749
5750 if (src_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005751 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005752 "memmove bogus src_offset %lu move len %lu dst len %lu",
5753 src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005754 BUG_ON(1);
5755 }
5756 if (dst_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005757 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005758 "memmove bogus dst_offset %lu move len %lu dst len %lu",
5759 dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005760 BUG_ON(1);
5761 }
5762
Chris Masond3977122009-01-05 21:25:51 -05005763 while (len > 0) {
Johannes Thumshirn70730172018-12-05 15:23:03 +01005764 dst_off_in_page = offset_in_page(start_offset + dst_offset);
5765 src_off_in_page = offset_in_page(start_offset + src_offset);
Chris Masond1310b22008-01-24 16:13:08 -05005766
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005767 dst_i = (start_offset + dst_offset) >> PAGE_SHIFT;
5768 src_i = (start_offset + src_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005769
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005770 cur = min(len, (unsigned long)(PAGE_SIZE -
Chris Masond1310b22008-01-24 16:13:08 -05005771 src_off_in_page));
5772 cur = min_t(unsigned long, cur,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005773 (unsigned long)(PAGE_SIZE - dst_off_in_page));
Chris Masond1310b22008-01-24 16:13:08 -05005774
David Sterbafb85fc92014-07-31 01:03:53 +02005775 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005776 dst_off_in_page, src_off_in_page, cur);
5777
5778 src_offset += cur;
5779 dst_offset += cur;
5780 len -= cur;
5781 }
5782}
Chris Masond1310b22008-01-24 16:13:08 -05005783
5784void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5785 unsigned long src_offset, unsigned long len)
5786{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005787 struct btrfs_fs_info *fs_info = dst->fs_info;
Chris Masond1310b22008-01-24 16:13:08 -05005788 size_t cur;
5789 size_t dst_off_in_page;
5790 size_t src_off_in_page;
5791 unsigned long dst_end = dst_offset + len - 1;
5792 unsigned long src_end = src_offset + len - 1;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005793 size_t start_offset = offset_in_page(dst->start);
Chris Masond1310b22008-01-24 16:13:08 -05005794 unsigned long dst_i;
5795 unsigned long src_i;
5796
5797 if (src_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005798 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005799 "memmove bogus src_offset %lu move len %lu len %lu",
5800 src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005801 BUG_ON(1);
5802 }
5803 if (dst_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005804 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005805 "memmove bogus dst_offset %lu move len %lu len %lu",
5806 dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005807 BUG_ON(1);
5808 }
Chris Mason727011e2010-08-06 13:21:20 -04005809 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005810 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5811 return;
5812 }
Chris Masond3977122009-01-05 21:25:51 -05005813 while (len > 0) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005814 dst_i = (start_offset + dst_end) >> PAGE_SHIFT;
5815 src_i = (start_offset + src_end) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005816
Johannes Thumshirn70730172018-12-05 15:23:03 +01005817 dst_off_in_page = offset_in_page(start_offset + dst_end);
5818 src_off_in_page = offset_in_page(start_offset + src_end);
Chris Masond1310b22008-01-24 16:13:08 -05005819
5820 cur = min_t(unsigned long, len, src_off_in_page + 1);
5821 cur = min(cur, dst_off_in_page + 1);
David Sterbafb85fc92014-07-31 01:03:53 +02005822 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005823 dst_off_in_page - cur + 1,
5824 src_off_in_page - cur + 1, cur);
5825
5826 dst_end -= cur;
5827 src_end -= cur;
5828 len -= cur;
5829 }
5830}
Chris Mason6af118ce2008-07-22 11:18:07 -04005831
David Sterbaf7a52a42013-04-26 14:56:29 +00005832int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005833{
Chris Mason6af118ce2008-07-22 11:18:07 -04005834 struct extent_buffer *eb;
Miao Xie897ca6e92010-10-26 20:57:29 -04005835
Miao Xie19fe0a82010-10-26 20:57:29 -04005836 /*
Nicholas D Steeves01327612016-05-19 21:18:45 -04005837 * We need to make sure nobody is attaching this page to an eb right
Josef Bacik3083ee22012-03-09 16:01:49 -05005838 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005839 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005840 spin_lock(&page->mapping->private_lock);
5841 if (!PagePrivate(page)) {
5842 spin_unlock(&page->mapping->private_lock);
5843 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005844 }
5845
Josef Bacik3083ee22012-03-09 16:01:49 -05005846 eb = (struct extent_buffer *)page->private;
5847 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005848
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005849 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005850 * This is a little awful but should be ok, we need to make sure that
5851 * the eb doesn't disappear out from under us while we're looking at
5852 * this page.
5853 */
5854 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005855 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05005856 spin_unlock(&eb->refs_lock);
5857 spin_unlock(&page->mapping->private_lock);
5858 return 0;
5859 }
5860 spin_unlock(&page->mapping->private_lock);
5861
Josef Bacik3083ee22012-03-09 16:01:49 -05005862 /*
5863 * If tree ref isn't set then we know the ref on this eb is a real ref,
5864 * so just return, this page will likely be freed soon anyway.
5865 */
5866 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5867 spin_unlock(&eb->refs_lock);
5868 return 0;
5869 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005870
David Sterbaf7a52a42013-04-26 14:56:29 +00005871 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005872}