blob: ca8b8e785cf3a17f963449058210d1f0ff4cfd0d [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
David Sterbac1d7c512018-04-03 19:23:33 +02002
Chris Masond1310b22008-01-24 16:13:08 -05003#include <linux/bitops.h>
4#include <linux/slab.h>
5#include <linux/bio.h>
6#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/pagemap.h>
8#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05009#include <linux/spinlock.h>
10#include <linux/blkdev.h>
11#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050012#include <linux/writeback.h>
13#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070014#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060015#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050016#include "extent_io.h"
17#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040018#include "ctree.h"
19#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020020#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010021#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040022#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040023#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080024#include "backref.h"
David Sterba6af49db2017-06-23 04:09:57 +020025#include "disk-io.h"
Chris Masond1310b22008-01-24 16:13:08 -050026
Chris Masond1310b22008-01-24 16:13:08 -050027static struct kmem_cache *extent_state_cache;
28static struct kmem_cache *extent_buffer_cache;
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -040029static struct bio_set btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050030
Filipe Manana27a35072014-07-06 20:09:59 +010031static inline bool extent_state_in_tree(const struct extent_state *state)
32{
33 return !RB_EMPTY_NODE(&state->rb_node);
34}
35
Eric Sandeen6d49ba12013-04-22 16:12:31 +000036#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050037static LIST_HEAD(buffers);
38static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040039
Chris Masond3977122009-01-05 21:25:51 -050040static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000041
42static inline
43void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
44{
45 unsigned long flags;
46
47 spin_lock_irqsave(&leak_lock, flags);
48 list_add(new, head);
49 spin_unlock_irqrestore(&leak_lock, flags);
50}
51
52static inline
53void btrfs_leak_debug_del(struct list_head *entry)
54{
55 unsigned long flags;
56
57 spin_lock_irqsave(&leak_lock, flags);
58 list_del(entry);
59 spin_unlock_irqrestore(&leak_lock, flags);
60}
61
62static inline
63void btrfs_leak_debug_check(void)
64{
65 struct extent_state *state;
66 struct extent_buffer *eb;
67
68 while (!list_empty(&states)) {
69 state = list_entry(states.next, struct extent_state, leak_list);
David Sterba9ee49a042015-01-14 19:52:13 +010070 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
Filipe Manana27a35072014-07-06 20:09:59 +010071 state->start, state->end, state->state,
72 extent_state_in_tree(state),
Elena Reshetovab7ac31b2017-03-03 10:55:19 +020073 refcount_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000074 list_del(&state->leak_list);
75 kmem_cache_free(extent_state_cache, state);
76 }
77
78 while (!list_empty(&buffers)) {
79 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Liu Boaf2679e2018-01-25 11:02:48 -070080 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
81 eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000082 list_del(&eb->leak_list);
83 kmem_cache_free(extent_buffer_cache, eb);
84 }
85}
David Sterba8d599ae2013-04-30 15:22:23 +000086
Josef Bacika5dee372013-12-13 10:02:44 -050087#define btrfs_debug_check_extent_io_range(tree, start, end) \
88 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000089static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050090 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000091{
Nikolay Borisov65a680f2018-11-01 14:09:49 +020092 struct inode *inode = tree->private_data;
93 u64 isize;
94
95 if (!inode || !is_data_inode(inode))
96 return;
97
98 isize = i_size_read(inode);
99 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
100 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
101 "%s: ino %llu isize %llu odd range [%llu,%llu]",
102 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
103 }
David Sterba8d599ae2013-04-30 15:22:23 +0000104}
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000105#else
106#define btrfs_leak_debug_add(new, head) do {} while (0)
107#define btrfs_leak_debug_del(entry) do {} while (0)
108#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000109#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400110#endif
Chris Masond1310b22008-01-24 16:13:08 -0500111
Chris Masond1310b22008-01-24 16:13:08 -0500112#define BUFFER_LRU_MAX 64
113
114struct tree_entry {
115 u64 start;
116 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500117 struct rb_node rb_node;
118};
119
120struct extent_page_data {
121 struct bio *bio;
122 struct extent_io_tree *tree;
Chris Mason771ed682008-11-06 22:02:51 -0500123 /* tells writepage not to lock the state bits for this range
124 * it still does the unlocking
125 */
Chris Masonffbd5172009-04-20 15:50:09 -0400126 unsigned int extent_locked:1;
127
Christoph Hellwig70fd7612016-11-01 07:40:10 -0600128 /* tells the submit_bio code to use REQ_SYNC */
Chris Masonffbd5172009-04-20 15:50:09 -0400129 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500130};
131
David Sterba57599c72018-03-01 17:56:34 +0100132static int add_extent_changeset(struct extent_state *state, unsigned bits,
Qu Wenruod38ed272015-10-12 14:53:37 +0800133 struct extent_changeset *changeset,
134 int set)
135{
136 int ret;
137
138 if (!changeset)
David Sterba57599c72018-03-01 17:56:34 +0100139 return 0;
Qu Wenruod38ed272015-10-12 14:53:37 +0800140 if (set && (state->state & bits) == bits)
David Sterba57599c72018-03-01 17:56:34 +0100141 return 0;
Qu Wenruofefdc552015-10-12 15:35:38 +0800142 if (!set && (state->state & bits) == 0)
David Sterba57599c72018-03-01 17:56:34 +0100143 return 0;
Qu Wenruod38ed272015-10-12 14:53:37 +0800144 changeset->bytes_changed += state->end - state->start + 1;
David Sterba53d32352017-02-13 13:42:29 +0100145 ret = ulist_add(&changeset->range_changed, state->start, state->end,
Qu Wenruod38ed272015-10-12 14:53:37 +0800146 GFP_ATOMIC);
David Sterba57599c72018-03-01 17:56:34 +0100147 return ret;
Qu Wenruod38ed272015-10-12 14:53:37 +0800148}
149
Qu Wenruobb58eb92019-01-25 13:09:15 +0800150static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
151 unsigned long bio_flags)
152{
153 blk_status_t ret = 0;
154 struct bio_vec *bvec = bio_last_bvec_all(bio);
Linus Torvalds80201fe2019-03-08 14:12:17 -0800155 struct bio_vec bv;
Qu Wenruobb58eb92019-01-25 13:09:15 +0800156 struct extent_io_tree *tree = bio->bi_private;
157 u64 start;
158
Linus Torvalds80201fe2019-03-08 14:12:17 -0800159 mp_bvec_last_segment(bvec, &bv);
160 start = page_offset(bv.bv_page) + bv.bv_offset;
Qu Wenruobb58eb92019-01-25 13:09:15 +0800161
162 bio->bi_private = NULL;
163
164 if (tree->ops)
165 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
166 mirror_num, bio_flags, start);
167 else
168 btrfsic_submit_bio(bio);
169
170 return blk_status_to_errno(ret);
171}
172
173static void flush_write_bio(struct extent_page_data *epd)
174{
175 if (epd->bio) {
176 int ret;
177
178 ret = submit_one_bio(epd->bio, 0, 0);
179 BUG_ON(ret < 0); /* -ENOMEM */
180 epd->bio = NULL;
181 }
182}
David Sterbae2932ee2017-06-23 04:16:17 +0200183
Chris Masond1310b22008-01-24 16:13:08 -0500184int __init extent_io_init(void)
185{
David Sterba837e1972012-09-07 03:00:48 -0600186 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200187 sizeof(struct extent_state), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300188 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500189 if (!extent_state_cache)
190 return -ENOMEM;
191
David Sterba837e1972012-09-07 03:00:48 -0600192 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200193 sizeof(struct extent_buffer), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300194 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500195 if (!extent_buffer_cache)
196 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400197
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400198 if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
199 offsetof(struct btrfs_io_bio, bio),
200 BIOSET_NEED_BVECS))
Chris Mason9be33952013-05-17 18:30:14 -0400201 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700202
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400203 if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700204 goto free_bioset;
205
Chris Masond1310b22008-01-24 16:13:08 -0500206 return 0;
207
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700208free_bioset:
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400209 bioset_exit(&btrfs_bioset);
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700210
Chris Mason9be33952013-05-17 18:30:14 -0400211free_buffer_cache:
212 kmem_cache_destroy(extent_buffer_cache);
213 extent_buffer_cache = NULL;
214
Chris Masond1310b22008-01-24 16:13:08 -0500215free_state_cache:
216 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400217 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500218 return -ENOMEM;
219}
220
David Sterbae67c7182018-02-19 17:24:18 +0100221void __cold extent_io_exit(void)
Chris Masond1310b22008-01-24 16:13:08 -0500222{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000223 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000224
225 /*
226 * Make sure all delayed rcu free are flushed before we
227 * destroy caches.
228 */
229 rcu_barrier();
Kinglong Mee5598e902016-01-29 21:36:35 +0800230 kmem_cache_destroy(extent_state_cache);
231 kmem_cache_destroy(extent_buffer_cache);
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400232 bioset_exit(&btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500233}
234
235void extent_io_tree_init(struct extent_io_tree *tree,
Josef Bacikc6100a42017-05-05 11:57:13 -0400236 void *private_data)
Chris Masond1310b22008-01-24 16:13:08 -0500237{
Eric Paris6bef4d32010-02-23 19:43:04 +0000238 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500239 tree->ops = NULL;
240 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500241 spin_lock_init(&tree->lock);
Josef Bacikc6100a42017-05-05 11:57:13 -0400242 tree->private_data = private_data;
Chris Masond1310b22008-01-24 16:13:08 -0500243}
Chris Masond1310b22008-01-24 16:13:08 -0500244
Christoph Hellwigb2950862008-12-02 09:54:17 -0500245static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500246{
247 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500248
Michal Hocko3ba7ab22017-01-09 15:39:02 +0100249 /*
250 * The given mask might be not appropriate for the slab allocator,
251 * drop the unsupported bits
252 */
253 mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
Chris Masond1310b22008-01-24 16:13:08 -0500254 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400255 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500256 return state;
257 state->state = 0;
David Sterba47dc1962016-02-11 13:24:13 +0100258 state->failrec = NULL;
Filipe Manana27a35072014-07-06 20:09:59 +0100259 RB_CLEAR_NODE(&state->rb_node);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000260 btrfs_leak_debug_add(&state->leak_list, &states);
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200261 refcount_set(&state->refs, 1);
Chris Masond1310b22008-01-24 16:13:08 -0500262 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100263 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500264 return state;
265}
Chris Masond1310b22008-01-24 16:13:08 -0500266
Chris Mason4845e442010-05-25 20:56:50 -0400267void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500268{
Chris Masond1310b22008-01-24 16:13:08 -0500269 if (!state)
270 return;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200271 if (refcount_dec_and_test(&state->refs)) {
Filipe Manana27a35072014-07-06 20:09:59 +0100272 WARN_ON(extent_state_in_tree(state));
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000273 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100274 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500275 kmem_cache_free(extent_state_cache, state);
276 }
277}
Chris Masond1310b22008-01-24 16:13:08 -0500278
Filipe Mananaf2071b22014-02-12 15:05:53 +0000279static struct rb_node *tree_insert(struct rb_root *root,
280 struct rb_node *search_start,
281 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000282 struct rb_node *node,
283 struct rb_node ***p_in,
284 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500285{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000286 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500287 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500288 struct tree_entry *entry;
289
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000290 if (p_in && parent_in) {
291 p = *p_in;
292 parent = *parent_in;
293 goto do_insert;
294 }
295
Filipe Mananaf2071b22014-02-12 15:05:53 +0000296 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500297 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500298 parent = *p;
299 entry = rb_entry(parent, struct tree_entry, rb_node);
300
301 if (offset < entry->start)
302 p = &(*p)->rb_left;
303 else if (offset > entry->end)
304 p = &(*p)->rb_right;
305 else
306 return parent;
307 }
308
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000309do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500310 rb_link_node(node, parent, p);
311 rb_insert_color(node, root);
312 return NULL;
313}
314
Chris Mason80ea96b2008-02-01 14:51:59 -0500315static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000316 struct rb_node **next_ret,
Nikolay Borisov352646c2019-01-30 16:51:00 +0200317 struct rb_node **prev_ret,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000318 struct rb_node ***p_ret,
319 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500320{
Chris Mason80ea96b2008-02-01 14:51:59 -0500321 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000322 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500323 struct rb_node *prev = NULL;
324 struct rb_node *orig_prev = NULL;
325 struct tree_entry *entry;
326 struct tree_entry *prev_entry = NULL;
327
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000328 while (*n) {
329 prev = *n;
330 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500331 prev_entry = entry;
332
333 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000334 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500335 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000336 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500337 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000338 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500339 }
340
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000341 if (p_ret)
342 *p_ret = n;
343 if (parent_ret)
344 *parent_ret = prev;
345
Nikolay Borisov352646c2019-01-30 16:51:00 +0200346 if (next_ret) {
Chris Masond1310b22008-01-24 16:13:08 -0500347 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500348 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500349 prev = rb_next(prev);
350 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
351 }
Nikolay Borisov352646c2019-01-30 16:51:00 +0200352 *next_ret = prev;
Chris Masond1310b22008-01-24 16:13:08 -0500353 prev = orig_prev;
354 }
355
Nikolay Borisov352646c2019-01-30 16:51:00 +0200356 if (prev_ret) {
Chris Masond1310b22008-01-24 16:13:08 -0500357 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500358 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500359 prev = rb_prev(prev);
360 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
361 }
Nikolay Borisov352646c2019-01-30 16:51:00 +0200362 *prev_ret = prev;
Chris Masond1310b22008-01-24 16:13:08 -0500363 }
364 return NULL;
365}
366
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000367static inline struct rb_node *
368tree_search_for_insert(struct extent_io_tree *tree,
369 u64 offset,
370 struct rb_node ***p_ret,
371 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500372{
Nikolay Borisov352646c2019-01-30 16:51:00 +0200373 struct rb_node *next= NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500374 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500375
Nikolay Borisov352646c2019-01-30 16:51:00 +0200376 ret = __etree_search(tree, offset, &next, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500377 if (!ret)
Nikolay Borisov352646c2019-01-30 16:51:00 +0200378 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500379 return ret;
380}
381
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000382static inline struct rb_node *tree_search(struct extent_io_tree *tree,
383 u64 offset)
384{
385 return tree_search_for_insert(tree, offset, NULL, NULL);
386}
387
Chris Masond1310b22008-01-24 16:13:08 -0500388/*
389 * utility function to look for merge candidates inside a given range.
390 * Any extents with matching state are merged together into a single
391 * extent in the tree. Extents with EXTENT_IO in their state field
392 * are not merged because the end_io handlers need to be able to do
393 * operations on them without sleeping (or doing allocations/splits).
394 *
395 * This should be called with the tree lock held.
396 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000397static void merge_state(struct extent_io_tree *tree,
398 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500399{
400 struct extent_state *other;
401 struct rb_node *other_node;
402
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400403 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000404 return;
Chris Masond1310b22008-01-24 16:13:08 -0500405
406 other_node = rb_prev(&state->rb_node);
407 if (other_node) {
408 other = rb_entry(other_node, struct extent_state, rb_node);
409 if (other->end == state->start - 1 &&
410 other->state == state->state) {
Nikolay Borisov5c848192018-11-01 14:09:52 +0200411 if (tree->private_data &&
412 is_data_inode(tree->private_data))
413 btrfs_merge_delalloc_extent(tree->private_data,
414 state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500415 state->start = other->start;
Chris Masond1310b22008-01-24 16:13:08 -0500416 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100417 RB_CLEAR_NODE(&other->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500418 free_extent_state(other);
419 }
420 }
421 other_node = rb_next(&state->rb_node);
422 if (other_node) {
423 other = rb_entry(other_node, struct extent_state, rb_node);
424 if (other->start == state->end + 1 &&
425 other->state == state->state) {
Nikolay Borisov5c848192018-11-01 14:09:52 +0200426 if (tree->private_data &&
427 is_data_inode(tree->private_data))
428 btrfs_merge_delalloc_extent(tree->private_data,
429 state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400430 state->end = other->end;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400431 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100432 RB_CLEAR_NODE(&other->rb_node);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400433 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500434 }
435 }
Chris Masond1310b22008-01-24 16:13:08 -0500436}
437
Xiao Guangrong3150b692011-07-14 03:19:08 +0000438static void set_state_bits(struct extent_io_tree *tree,
Qu Wenruod38ed272015-10-12 14:53:37 +0800439 struct extent_state *state, unsigned *bits,
440 struct extent_changeset *changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000441
Chris Masond1310b22008-01-24 16:13:08 -0500442/*
443 * insert an extent_state struct into the tree. 'bits' are set on the
444 * struct before it is inserted.
445 *
446 * This may return -EEXIST if the extent is already there, in which case the
447 * state struct is freed.
448 *
449 * The tree lock is not taken internally. This is a utility function and
450 * probably isn't what you want to call (see set/clear_extent_bit).
451 */
452static int insert_state(struct extent_io_tree *tree,
453 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000454 struct rb_node ***p,
455 struct rb_node **parent,
Qu Wenruod38ed272015-10-12 14:53:37 +0800456 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500457{
458 struct rb_node *node;
459
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000460 if (end < start)
Frank Holtonefe120a2013-12-20 11:37:06 -0500461 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200462 end, start);
Chris Masond1310b22008-01-24 16:13:08 -0500463 state->start = start;
464 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400465
Qu Wenruod38ed272015-10-12 14:53:37 +0800466 set_state_bits(tree, state, bits, changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000467
Filipe Mananaf2071b22014-02-12 15:05:53 +0000468 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500469 if (node) {
470 struct extent_state *found;
471 found = rb_entry(node, struct extent_state, rb_node);
Jeff Mahoney62e85572016-09-20 10:05:01 -0400472 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200473 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500474 return -EEXIST;
475 }
476 merge_state(tree, state);
477 return 0;
478}
479
480/*
481 * split a given extent state struct in two, inserting the preallocated
482 * struct 'prealloc' as the newly created second half. 'split' indicates an
483 * offset inside 'orig' where it should be split.
484 *
485 * Before calling,
486 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
487 * are two extent state structs in the tree:
488 * prealloc: [orig->start, split - 1]
489 * orig: [ split, orig->end ]
490 *
491 * The tree locks are not taken by this function. They need to be held
492 * by the caller.
493 */
494static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
495 struct extent_state *prealloc, u64 split)
496{
497 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400498
Nikolay Borisovabbb55f2018-11-01 14:09:53 +0200499 if (tree->private_data && is_data_inode(tree->private_data))
500 btrfs_split_delalloc_extent(tree->private_data, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400501
Chris Masond1310b22008-01-24 16:13:08 -0500502 prealloc->start = orig->start;
503 prealloc->end = split - 1;
504 prealloc->state = orig->state;
505 orig->start = split;
506
Filipe Mananaf2071b22014-02-12 15:05:53 +0000507 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
508 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500509 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500510 free_extent_state(prealloc);
511 return -EEXIST;
512 }
513 return 0;
514}
515
Li Zefancdc6a392012-03-12 16:39:48 +0800516static struct extent_state *next_state(struct extent_state *state)
517{
518 struct rb_node *next = rb_next(&state->rb_node);
519 if (next)
520 return rb_entry(next, struct extent_state, rb_node);
521 else
522 return NULL;
523}
524
Chris Masond1310b22008-01-24 16:13:08 -0500525/*
526 * utility function to clear some bits in an extent state struct.
Andrea Gelmini52042d82018-11-28 12:05:13 +0100527 * it will optionally wake up anyone waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500528 *
529 * If no bits are set on the state struct after clearing things, the
530 * struct is freed and removed from the tree
531 */
Li Zefancdc6a392012-03-12 16:39:48 +0800532static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
533 struct extent_state *state,
Qu Wenruofefdc552015-10-12 15:35:38 +0800534 unsigned *bits, int wake,
535 struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500536{
Li Zefancdc6a392012-03-12 16:39:48 +0800537 struct extent_state *next;
David Sterba9ee49a042015-01-14 19:52:13 +0100538 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
David Sterba57599c72018-03-01 17:56:34 +0100539 int ret;
Chris Masond1310b22008-01-24 16:13:08 -0500540
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400541 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500542 u64 range = state->end - state->start + 1;
543 WARN_ON(range > tree->dirty_bytes);
544 tree->dirty_bytes -= range;
545 }
Nikolay Borisova36bb5f2018-11-01 14:09:51 +0200546
547 if (tree->private_data && is_data_inode(tree->private_data))
548 btrfs_clear_delalloc_extent(tree->private_data, state, bits);
549
David Sterba57599c72018-03-01 17:56:34 +0100550 ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
551 BUG_ON(ret < 0);
Josef Bacik32c00af2009-10-08 13:34:05 -0400552 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500553 if (wake)
554 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400555 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800556 next = next_state(state);
Filipe Manana27a35072014-07-06 20:09:59 +0100557 if (extent_state_in_tree(state)) {
Chris Masond1310b22008-01-24 16:13:08 -0500558 rb_erase(&state->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100559 RB_CLEAR_NODE(&state->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500560 free_extent_state(state);
561 } else {
562 WARN_ON(1);
563 }
564 } else {
565 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800566 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500567 }
Li Zefancdc6a392012-03-12 16:39:48 +0800568 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500569}
570
Xiao Guangrong82337672011-04-20 06:44:57 +0000571static struct extent_state *
572alloc_extent_state_atomic(struct extent_state *prealloc)
573{
574 if (!prealloc)
575 prealloc = alloc_extent_state(GFP_ATOMIC);
576
577 return prealloc;
578}
579
Eric Sandeen48a3b632013-04-25 20:41:01 +0000580static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400581{
David Sterba05912a32018-07-18 19:23:45 +0200582 struct inode *inode = tree->private_data;
583
584 btrfs_panic(btrfs_sb(inode->i_sb), err,
585 "locking error: extent tree was modified by another thread while locked");
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400586}
587
Chris Masond1310b22008-01-24 16:13:08 -0500588/*
589 * clear some bits on a range in the tree. This may require splitting
590 * or inserting elements in the tree, so the gfp mask is used to
591 * indicate which allocations or sleeping are allowed.
592 *
593 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
594 * the given range from the tree regardless of state (ie for truncate).
595 *
596 * the range [start, end] is inclusive.
597 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100598 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500599 */
David Sterba66b0c882017-10-31 16:30:47 +0100600int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
Qu Wenruofefdc552015-10-12 15:35:38 +0800601 unsigned bits, int wake, int delete,
602 struct extent_state **cached_state,
603 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500604{
605 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400606 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500607 struct extent_state *prealloc = NULL;
608 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400609 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500610 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000611 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500612
Josef Bacika5dee372013-12-13 10:02:44 -0500613 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000614
Josef Bacik7ee9e442013-06-21 16:37:03 -0400615 if (bits & EXTENT_DELALLOC)
616 bits |= EXTENT_NORESERVE;
617
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400618 if (delete)
619 bits |= ~EXTENT_CTLBITS;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400620
Josef Bacik2ac55d42010-02-03 19:33:23 +0000621 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
622 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500623again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800624 if (!prealloc && gfpflags_allow_blocking(mask)) {
Filipe Mananac7bc6312014-11-03 14:12:57 +0000625 /*
626 * Don't care for allocation failure here because we might end
627 * up not needing the pre-allocated extent state at all, which
628 * is the case if we only have in the tree extent states that
629 * cover our input range and don't cover too any other range.
630 * If we end up needing a new extent state we allocate it later.
631 */
Chris Masond1310b22008-01-24 16:13:08 -0500632 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500633 }
634
Chris Masoncad321a2008-12-17 14:51:42 -0500635 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400636 if (cached_state) {
637 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000638
639 if (clear) {
640 *cached_state = NULL;
641 cached_state = NULL;
642 }
643
Filipe Manana27a35072014-07-06 20:09:59 +0100644 if (cached && extent_state_in_tree(cached) &&
645 cached->start <= start && cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000646 if (clear)
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200647 refcount_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400648 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400649 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400650 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000651 if (clear)
652 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400653 }
Chris Masond1310b22008-01-24 16:13:08 -0500654 /*
655 * this search will find the extents that end after
656 * our range starts
657 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500658 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500659 if (!node)
660 goto out;
661 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400662hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500663 if (state->start > end)
664 goto out;
665 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400666 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500667
Liu Bo04493142012-02-16 18:34:37 +0800668 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800669 if (!(state->state & bits)) {
670 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800671 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800672 }
Liu Bo04493142012-02-16 18:34:37 +0800673
Chris Masond1310b22008-01-24 16:13:08 -0500674 /*
675 * | ---- desired range ---- |
676 * | state | or
677 * | ------------- state -------------- |
678 *
679 * We need to split the extent we found, and may flip
680 * bits on second half.
681 *
682 * If the extent we found extends past our range, we
683 * just split and search again. It'll get split again
684 * the next time though.
685 *
686 * If the extent we found is inside our range, we clear
687 * the desired bit on it.
688 */
689
690 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000691 prealloc = alloc_extent_state_atomic(prealloc);
692 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500693 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400694 if (err)
695 extent_io_tree_panic(tree, err);
696
Chris Masond1310b22008-01-24 16:13:08 -0500697 prealloc = NULL;
698 if (err)
699 goto out;
700 if (state->end <= end) {
Qu Wenruofefdc552015-10-12 15:35:38 +0800701 state = clear_state_bit(tree, state, &bits, wake,
702 changeset);
Liu Bod1ac6e42012-05-10 18:10:39 +0800703 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500704 }
705 goto search_again;
706 }
707 /*
708 * | ---- desired range ---- |
709 * | state |
710 * We need to split the extent, and clear the bit
711 * on the first half
712 */
713 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000714 prealloc = alloc_extent_state_atomic(prealloc);
715 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500716 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400717 if (err)
718 extent_io_tree_panic(tree, err);
719
Chris Masond1310b22008-01-24 16:13:08 -0500720 if (wake)
721 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400722
Qu Wenruofefdc552015-10-12 15:35:38 +0800723 clear_state_bit(tree, prealloc, &bits, wake, changeset);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400724
Chris Masond1310b22008-01-24 16:13:08 -0500725 prealloc = NULL;
726 goto out;
727 }
Chris Mason42daec22009-09-23 19:51:09 -0400728
Qu Wenruofefdc552015-10-12 15:35:38 +0800729 state = clear_state_bit(tree, state, &bits, wake, changeset);
Liu Bo04493142012-02-16 18:34:37 +0800730next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400731 if (last_end == (u64)-1)
732 goto out;
733 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800734 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800735 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500736
737search_again:
738 if (start > end)
739 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500740 spin_unlock(&tree->lock);
Mel Gormand0164ad2015-11-06 16:28:21 -0800741 if (gfpflags_allow_blocking(mask))
Chris Masond1310b22008-01-24 16:13:08 -0500742 cond_resched();
743 goto again;
David Sterba7ab5cb22016-04-27 01:02:15 +0200744
745out:
746 spin_unlock(&tree->lock);
747 if (prealloc)
748 free_extent_state(prealloc);
749
750 return 0;
751
Chris Masond1310b22008-01-24 16:13:08 -0500752}
Chris Masond1310b22008-01-24 16:13:08 -0500753
Jeff Mahoney143bede2012-03-01 14:56:26 +0100754static void wait_on_state(struct extent_io_tree *tree,
755 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500756 __releases(tree->lock)
757 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500758{
759 DEFINE_WAIT(wait);
760 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500761 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500762 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500763 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500764 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500765}
766
767/*
768 * waits for one or more bits to clear on a range in the state tree.
769 * The range [start, end] is inclusive.
770 * The tree lock is taken by this function
771 */
David Sterba41074882013-04-29 13:38:46 +0000772static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
773 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500774{
775 struct extent_state *state;
776 struct rb_node *node;
777
Josef Bacika5dee372013-12-13 10:02:44 -0500778 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000779
Chris Masoncad321a2008-12-17 14:51:42 -0500780 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500781again:
782 while (1) {
783 /*
784 * this search will find all the extents that end after
785 * our range starts
786 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500787 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100788process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500789 if (!node)
790 break;
791
792 state = rb_entry(node, struct extent_state, rb_node);
793
794 if (state->start > end)
795 goto out;
796
797 if (state->state & bits) {
798 start = state->start;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200799 refcount_inc(&state->refs);
Chris Masond1310b22008-01-24 16:13:08 -0500800 wait_on_state(tree, state);
801 free_extent_state(state);
802 goto again;
803 }
804 start = state->end + 1;
805
806 if (start > end)
807 break;
808
Filipe Mananac50d3e72014-03-31 14:53:25 +0100809 if (!cond_resched_lock(&tree->lock)) {
810 node = rb_next(node);
811 goto process_node;
812 }
Chris Masond1310b22008-01-24 16:13:08 -0500813 }
814out:
Chris Masoncad321a2008-12-17 14:51:42 -0500815 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500816}
Chris Masond1310b22008-01-24 16:13:08 -0500817
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000818static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500819 struct extent_state *state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800820 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500821{
David Sterba9ee49a042015-01-14 19:52:13 +0100822 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
David Sterba57599c72018-03-01 17:56:34 +0100823 int ret;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400824
Nikolay Borisove06a1fc2018-11-01 14:09:50 +0200825 if (tree->private_data && is_data_inode(tree->private_data))
826 btrfs_set_delalloc_extent(tree->private_data, state, bits);
827
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400828 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500829 u64 range = state->end - state->start + 1;
830 tree->dirty_bytes += range;
831 }
David Sterba57599c72018-03-01 17:56:34 +0100832 ret = add_extent_changeset(state, bits_to_set, changeset, 1);
833 BUG_ON(ret < 0);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400834 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500835}
836
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100837static void cache_state_if_flags(struct extent_state *state,
838 struct extent_state **cached_ptr,
David Sterba9ee49a042015-01-14 19:52:13 +0100839 unsigned flags)
Chris Mason2c64c532009-09-02 15:04:12 -0400840{
841 if (cached_ptr && !(*cached_ptr)) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100842 if (!flags || (state->state & flags)) {
Chris Mason2c64c532009-09-02 15:04:12 -0400843 *cached_ptr = state;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200844 refcount_inc(&state->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400845 }
846 }
847}
848
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100849static void cache_state(struct extent_state *state,
850 struct extent_state **cached_ptr)
851{
852 return cache_state_if_flags(state, cached_ptr,
853 EXTENT_IOBITS | EXTENT_BOUNDARY);
854}
855
Chris Masond1310b22008-01-24 16:13:08 -0500856/*
Chris Mason1edbb732009-09-02 13:24:36 -0400857 * set some bits on a range in the tree. This may require allocations or
858 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500859 *
Chris Mason1edbb732009-09-02 13:24:36 -0400860 * If any of the exclusive bits are set, this will fail with -EEXIST if some
861 * part of the range already has the desired bits set. The start of the
862 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500863 *
Chris Mason1edbb732009-09-02 13:24:36 -0400864 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500865 */
Chris Mason1edbb732009-09-02 13:24:36 -0400866
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100867static int __must_check
868__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100869 unsigned bits, unsigned exclusive_bits,
David Sterba41074882013-04-29 13:38:46 +0000870 u64 *failed_start, struct extent_state **cached_state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800871 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500872{
873 struct extent_state *state;
874 struct extent_state *prealloc = NULL;
875 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000876 struct rb_node **p;
877 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500878 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500879 u64 last_start;
880 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400881
Josef Bacika5dee372013-12-13 10:02:44 -0500882 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000883
Chris Masond1310b22008-01-24 16:13:08 -0500884again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800885 if (!prealloc && gfpflags_allow_blocking(mask)) {
David Sterba059f7912016-04-27 01:03:45 +0200886 /*
887 * Don't care for allocation failure here because we might end
888 * up not needing the pre-allocated extent state at all, which
889 * is the case if we only have in the tree extent states that
890 * cover our input range and don't cover too any other range.
891 * If we end up needing a new extent state we allocate it later.
892 */
Chris Masond1310b22008-01-24 16:13:08 -0500893 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500894 }
895
Chris Masoncad321a2008-12-17 14:51:42 -0500896 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400897 if (cached_state && *cached_state) {
898 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400899 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +0100900 extent_state_in_tree(state)) {
Chris Mason9655d292009-09-02 15:22:30 -0400901 node = &state->rb_node;
902 goto hit_next;
903 }
904 }
Chris Masond1310b22008-01-24 16:13:08 -0500905 /*
906 * this search will find all the extents that end after
907 * our range starts.
908 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000909 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -0500910 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000911 prealloc = alloc_extent_state_atomic(prealloc);
912 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000913 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +0800914 &p, &parent, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400915 if (err)
916 extent_io_tree_panic(tree, err);
917
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +0000918 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500919 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500920 goto out;
921 }
Chris Masond1310b22008-01-24 16:13:08 -0500922 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400923hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500924 last_start = state->start;
925 last_end = state->end;
926
927 /*
928 * | ---- desired range ---- |
929 * | state |
930 *
931 * Just lock what we found and keep going
932 */
933 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400934 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500935 *failed_start = state->start;
936 err = -EEXIST;
937 goto out;
938 }
Chris Mason42daec22009-09-23 19:51:09 -0400939
Qu Wenruod38ed272015-10-12 14:53:37 +0800940 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -0400941 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500942 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400943 if (last_end == (u64)-1)
944 goto out;
945 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800946 state = next_state(state);
947 if (start < end && state && state->start == start &&
948 !need_resched())
949 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500950 goto search_again;
951 }
952
953 /*
954 * | ---- desired range ---- |
955 * | state |
956 * or
957 * | ------------- state -------------- |
958 *
959 * We need to split the extent we found, and may flip bits on
960 * second half.
961 *
962 * If the extent we found extends past our
963 * range, we just split and search again. It'll get split
964 * again the next time though.
965 *
966 * If the extent we found is inside our range, we set the
967 * desired bit on it.
968 */
969 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400970 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500971 *failed_start = start;
972 err = -EEXIST;
973 goto out;
974 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000975
976 prealloc = alloc_extent_state_atomic(prealloc);
977 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500978 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400979 if (err)
980 extent_io_tree_panic(tree, err);
981
Chris Masond1310b22008-01-24 16:13:08 -0500982 prealloc = NULL;
983 if (err)
984 goto out;
985 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +0800986 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -0400987 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500988 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400989 if (last_end == (u64)-1)
990 goto out;
991 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800992 state = next_state(state);
993 if (start < end && state && state->start == start &&
994 !need_resched())
995 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500996 }
997 goto search_again;
998 }
999 /*
1000 * | ---- desired range ---- |
1001 * | state | or | state |
1002 *
1003 * There's a hole, we need to insert something in it and
1004 * ignore the extent we found.
1005 */
1006 if (state->start > start) {
1007 u64 this_end;
1008 if (end < last_start)
1009 this_end = end;
1010 else
Chris Masond3977122009-01-05 21:25:51 -05001011 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +00001012
1013 prealloc = alloc_extent_state_atomic(prealloc);
1014 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +00001015
1016 /*
1017 * Avoid to free 'prealloc' if it can be merged with
1018 * the later extent.
1019 */
Chris Masond1310b22008-01-24 16:13:08 -05001020 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001021 NULL, NULL, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001022 if (err)
1023 extent_io_tree_panic(tree, err);
1024
Chris Mason2c64c532009-09-02 15:04:12 -04001025 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001026 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05001027 start = this_end + 1;
1028 goto search_again;
1029 }
1030 /*
1031 * | ---- desired range ---- |
1032 * | state |
1033 * We need to split the extent, and set the bit
1034 * on the first half
1035 */
1036 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -04001037 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001038 *failed_start = start;
1039 err = -EEXIST;
1040 goto out;
1041 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001042
1043 prealloc = alloc_extent_state_atomic(prealloc);
1044 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001045 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001046 if (err)
1047 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -05001048
Qu Wenruod38ed272015-10-12 14:53:37 +08001049 set_state_bits(tree, prealloc, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -04001050 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001051 merge_state(tree, prealloc);
1052 prealloc = NULL;
1053 goto out;
1054 }
1055
David Sterbab5a4ba142016-04-27 01:02:15 +02001056search_again:
1057 if (start > end)
1058 goto out;
1059 spin_unlock(&tree->lock);
1060 if (gfpflags_allow_blocking(mask))
1061 cond_resched();
1062 goto again;
Chris Masond1310b22008-01-24 16:13:08 -05001063
1064out:
Chris Masoncad321a2008-12-17 14:51:42 -05001065 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001066 if (prealloc)
1067 free_extent_state(prealloc);
1068
1069 return err;
1070
Chris Masond1310b22008-01-24 16:13:08 -05001071}
Chris Masond1310b22008-01-24 16:13:08 -05001072
David Sterba41074882013-04-29 13:38:46 +00001073int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001074 unsigned bits, u64 * failed_start,
David Sterba41074882013-04-29 13:38:46 +00001075 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001076{
1077 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001078 cached_state, mask, NULL);
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001079}
1080
1081
Josef Bacik462d6fa2011-09-26 13:56:12 -04001082/**
Liu Bo10983f22012-07-11 15:26:19 +08001083 * convert_extent_bit - convert all bits in a given range from one bit to
1084 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001085 * @tree: the io tree to search
1086 * @start: the start offset in bytes
1087 * @end: the end offset in bytes (inclusive)
1088 * @bits: the bits to set in this range
1089 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001090 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001091 *
1092 * This will go through and set bits for the given range. If any states exist
1093 * already in this range they are set with the given bit and cleared of the
1094 * clear_bits. This is only meant to be used by things that are mergeable, ie
1095 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1096 * boundary bits like LOCK.
David Sterba210aa272016-04-26 23:54:39 +02001097 *
1098 * All allocations are done with GFP_NOFS.
Josef Bacik462d6fa2011-09-26 13:56:12 -04001099 */
1100int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001101 unsigned bits, unsigned clear_bits,
David Sterba210aa272016-04-26 23:54:39 +02001102 struct extent_state **cached_state)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001103{
1104 struct extent_state *state;
1105 struct extent_state *prealloc = NULL;
1106 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001107 struct rb_node **p;
1108 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001109 int err = 0;
1110 u64 last_start;
1111 u64 last_end;
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001112 bool first_iteration = true;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001113
Josef Bacika5dee372013-12-13 10:02:44 -05001114 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +00001115
Josef Bacik462d6fa2011-09-26 13:56:12 -04001116again:
David Sterba210aa272016-04-26 23:54:39 +02001117 if (!prealloc) {
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001118 /*
1119 * Best effort, don't worry if extent state allocation fails
1120 * here for the first iteration. We might have a cached state
1121 * that matches exactly the target range, in which case no
1122 * extent state allocations are needed. We'll only know this
1123 * after locking the tree.
1124 */
David Sterba210aa272016-04-26 23:54:39 +02001125 prealloc = alloc_extent_state(GFP_NOFS);
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001126 if (!prealloc && !first_iteration)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001127 return -ENOMEM;
1128 }
1129
1130 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001131 if (cached_state && *cached_state) {
1132 state = *cached_state;
1133 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +01001134 extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001135 node = &state->rb_node;
1136 goto hit_next;
1137 }
1138 }
1139
Josef Bacik462d6fa2011-09-26 13:56:12 -04001140 /*
1141 * this search will find all the extents that end after
1142 * our range starts.
1143 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001144 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001145 if (!node) {
1146 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001147 if (!prealloc) {
1148 err = -ENOMEM;
1149 goto out;
1150 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001151 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001152 &p, &parent, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001153 if (err)
1154 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001155 cache_state(prealloc, cached_state);
1156 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001157 goto out;
1158 }
1159 state = rb_entry(node, struct extent_state, rb_node);
1160hit_next:
1161 last_start = state->start;
1162 last_end = state->end;
1163
1164 /*
1165 * | ---- desired range ---- |
1166 * | state |
1167 *
1168 * Just lock what we found and keep going
1169 */
1170 if (state->start == start && state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001171 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001172 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001173 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001174 if (last_end == (u64)-1)
1175 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001176 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001177 if (start < end && state && state->start == start &&
1178 !need_resched())
1179 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001180 goto search_again;
1181 }
1182
1183 /*
1184 * | ---- desired range ---- |
1185 * | state |
1186 * or
1187 * | ------------- state -------------- |
1188 *
1189 * We need to split the extent we found, and may flip bits on
1190 * second half.
1191 *
1192 * If the extent we found extends past our
1193 * range, we just split and search again. It'll get split
1194 * again the next time though.
1195 *
1196 * If the extent we found is inside our range, we set the
1197 * desired bit on it.
1198 */
1199 if (state->start < start) {
1200 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001201 if (!prealloc) {
1202 err = -ENOMEM;
1203 goto out;
1204 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001205 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001206 if (err)
1207 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001208 prealloc = NULL;
1209 if (err)
1210 goto out;
1211 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001212 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001213 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001214 state = clear_state_bit(tree, state, &clear_bits, 0,
1215 NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001216 if (last_end == (u64)-1)
1217 goto out;
1218 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001219 if (start < end && state && state->start == start &&
1220 !need_resched())
1221 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001222 }
1223 goto search_again;
1224 }
1225 /*
1226 * | ---- desired range ---- |
1227 * | state | or | state |
1228 *
1229 * There's a hole, we need to insert something in it and
1230 * ignore the extent we found.
1231 */
1232 if (state->start > start) {
1233 u64 this_end;
1234 if (end < last_start)
1235 this_end = end;
1236 else
1237 this_end = last_start - 1;
1238
1239 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001240 if (!prealloc) {
1241 err = -ENOMEM;
1242 goto out;
1243 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001244
1245 /*
1246 * Avoid to free 'prealloc' if it can be merged with
1247 * the later extent.
1248 */
1249 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001250 NULL, NULL, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001251 if (err)
1252 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001253 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001254 prealloc = NULL;
1255 start = this_end + 1;
1256 goto search_again;
1257 }
1258 /*
1259 * | ---- desired range ---- |
1260 * | state |
1261 * We need to split the extent, and set the bit
1262 * on the first half
1263 */
1264 if (state->start <= end && state->end > end) {
1265 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001266 if (!prealloc) {
1267 err = -ENOMEM;
1268 goto out;
1269 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001270
1271 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001272 if (err)
1273 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001274
Qu Wenruod38ed272015-10-12 14:53:37 +08001275 set_state_bits(tree, prealloc, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001276 cache_state(prealloc, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001277 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001278 prealloc = NULL;
1279 goto out;
1280 }
1281
Josef Bacik462d6fa2011-09-26 13:56:12 -04001282search_again:
1283 if (start > end)
1284 goto out;
1285 spin_unlock(&tree->lock);
David Sterba210aa272016-04-26 23:54:39 +02001286 cond_resched();
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001287 first_iteration = false;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001288 goto again;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001289
1290out:
1291 spin_unlock(&tree->lock);
1292 if (prealloc)
1293 free_extent_state(prealloc);
1294
1295 return err;
1296}
1297
Chris Masond1310b22008-01-24 16:13:08 -05001298/* wrappers around set/clear extent bit */
Qu Wenruod38ed272015-10-12 14:53:37 +08001299int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba2c53b912016-04-26 23:54:39 +02001300 unsigned bits, struct extent_changeset *changeset)
Qu Wenruod38ed272015-10-12 14:53:37 +08001301{
1302 /*
1303 * We don't support EXTENT_LOCKED yet, as current changeset will
1304 * record any bits changed, so for EXTENT_LOCKED case, it will
1305 * either fail with -EEXIST or changeset will record the whole
1306 * range.
1307 */
1308 BUG_ON(bits & EXTENT_LOCKED);
1309
David Sterba2c53b912016-04-26 23:54:39 +02001310 return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
Qu Wenruod38ed272015-10-12 14:53:37 +08001311 changeset);
1312}
1313
Qu Wenruofefdc552015-10-12 15:35:38 +08001314int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1315 unsigned bits, int wake, int delete,
David Sterbaae0f1622017-10-31 16:37:52 +01001316 struct extent_state **cached)
Qu Wenruofefdc552015-10-12 15:35:38 +08001317{
1318 return __clear_extent_bit(tree, start, end, bits, wake, delete,
David Sterbaae0f1622017-10-31 16:37:52 +01001319 cached, GFP_NOFS, NULL);
Qu Wenruofefdc552015-10-12 15:35:38 +08001320}
1321
Qu Wenruofefdc552015-10-12 15:35:38 +08001322int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaf734c442016-04-26 23:54:39 +02001323 unsigned bits, struct extent_changeset *changeset)
Qu Wenruofefdc552015-10-12 15:35:38 +08001324{
1325 /*
1326 * Don't support EXTENT_LOCKED case, same reason as
1327 * set_record_extent_bits().
1328 */
1329 BUG_ON(bits & EXTENT_LOCKED);
1330
David Sterbaf734c442016-04-26 23:54:39 +02001331 return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
Qu Wenruofefdc552015-10-12 15:35:38 +08001332 changeset);
1333}
1334
Chris Masond352ac62008-09-29 15:18:18 -04001335/*
1336 * either insert or lock state struct between start and end use mask to tell
1337 * us if waiting is desired.
1338 */
Chris Mason1edbb732009-09-02 13:24:36 -04001339int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaff13db42015-12-03 14:30:40 +01001340 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001341{
1342 int err;
1343 u64 failed_start;
David Sterba9ee49a042015-01-14 19:52:13 +01001344
Chris Masond1310b22008-01-24 16:13:08 -05001345 while (1) {
David Sterbaff13db42015-12-03 14:30:40 +01001346 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001347 EXTENT_LOCKED, &failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001348 cached_state, GFP_NOFS, NULL);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001349 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001350 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1351 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001352 } else
Chris Masond1310b22008-01-24 16:13:08 -05001353 break;
Chris Masond1310b22008-01-24 16:13:08 -05001354 WARN_ON(start > end);
1355 }
1356 return err;
1357}
Chris Masond1310b22008-01-24 16:13:08 -05001358
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001359int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001360{
1361 int err;
1362 u64 failed_start;
1363
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001364 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
Qu Wenruod38ed272015-10-12 14:53:37 +08001365 &failed_start, NULL, GFP_NOFS, NULL);
Yan Zheng66435582008-10-30 14:19:50 -04001366 if (err == -EEXIST) {
1367 if (failed_start > start)
1368 clear_extent_bit(tree, start, failed_start - 1,
David Sterbaae0f1622017-10-31 16:37:52 +01001369 EXTENT_LOCKED, 1, 0, NULL);
Josef Bacik25179202008-10-29 14:49:05 -04001370 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001371 }
Josef Bacik25179202008-10-29 14:49:05 -04001372 return 1;
1373}
Josef Bacik25179202008-10-29 14:49:05 -04001374
David Sterbabd1fa4f2015-12-03 13:08:59 +01001375void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001376{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001377 unsigned long index = start >> PAGE_SHIFT;
1378 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001379 struct page *page;
1380
1381 while (index <= end_index) {
1382 page = find_get_page(inode->i_mapping, index);
1383 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1384 clear_page_dirty_for_io(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001385 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001386 index++;
1387 }
Chris Mason4adaa612013-03-26 13:07:00 -04001388}
1389
David Sterbaf6311572015-12-03 13:08:59 +01001390void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001391{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001392 unsigned long index = start >> PAGE_SHIFT;
1393 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001394 struct page *page;
1395
1396 while (index <= end_index) {
1397 page = find_get_page(inode->i_mapping, index);
1398 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Mason4adaa612013-03-26 13:07:00 -04001399 __set_page_dirty_nobuffers(page);
Konstantin Khebnikov8d386332015-02-11 15:26:55 -08001400 account_page_redirty(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001401 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001402 index++;
1403 }
Chris Mason4adaa612013-03-26 13:07:00 -04001404}
1405
Chris Masond352ac62008-09-29 15:18:18 -04001406/* find the first state struct with 'bits' set after 'start', and
1407 * return it. tree->lock must be held. NULL will returned if
1408 * nothing was found after 'start'
1409 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001410static struct extent_state *
1411find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +01001412 u64 start, unsigned bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001413{
1414 struct rb_node *node;
1415 struct extent_state *state;
1416
1417 /*
1418 * this search will find all the extents that end after
1419 * our range starts.
1420 */
1421 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001422 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001423 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001424
Chris Masond3977122009-01-05 21:25:51 -05001425 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001426 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001427 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001428 return state;
Chris Masond3977122009-01-05 21:25:51 -05001429
Chris Masond7fc6402008-02-18 12:12:38 -05001430 node = rb_next(node);
1431 if (!node)
1432 break;
1433 }
1434out:
1435 return NULL;
1436}
Chris Masond7fc6402008-02-18 12:12:38 -05001437
Chris Masond352ac62008-09-29 15:18:18 -04001438/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001439 * find the first offset in the io tree with 'bits' set. zero is
1440 * returned if we find something, and *start_ret and *end_ret are
1441 * set to reflect the state struct that was found.
1442 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001443 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001444 */
1445int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba9ee49a042015-01-14 19:52:13 +01001446 u64 *start_ret, u64 *end_ret, unsigned bits,
Josef Bacike6138872012-09-27 17:07:30 -04001447 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001448{
1449 struct extent_state *state;
1450 int ret = 1;
1451
1452 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001453 if (cached_state && *cached_state) {
1454 state = *cached_state;
Filipe Manana27a35072014-07-06 20:09:59 +01001455 if (state->end == start - 1 && extent_state_in_tree(state)) {
Liu Bo9688e9a2018-08-23 03:14:53 +08001456 while ((state = next_state(state)) != NULL) {
Josef Bacike6138872012-09-27 17:07:30 -04001457 if (state->state & bits)
1458 goto got_it;
Josef Bacike6138872012-09-27 17:07:30 -04001459 }
1460 free_extent_state(*cached_state);
1461 *cached_state = NULL;
1462 goto out;
1463 }
1464 free_extent_state(*cached_state);
1465 *cached_state = NULL;
1466 }
1467
Xiao Guangrong69261c42011-07-14 03:19:45 +00001468 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001469got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001470 if (state) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +01001471 cache_state_if_flags(state, cached_state, 0);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001472 *start_ret = state->start;
1473 *end_ret = state->end;
1474 ret = 0;
1475 }
Josef Bacike6138872012-09-27 17:07:30 -04001476out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001477 spin_unlock(&tree->lock);
1478 return ret;
1479}
1480
1481/*
Chris Masond352ac62008-09-29 15:18:18 -04001482 * find a contiguous range of bytes in the file marked as delalloc, not
1483 * more than 'max_bytes'. start and end are used to return the range,
1484 *
Lu Fengqi3522e902018-11-29 11:33:38 +08001485 * true is returned if we find something, false if nothing was in the tree
Chris Masond352ac62008-09-29 15:18:18 -04001486 */
Lu Fengqi3522e902018-11-29 11:33:38 +08001487static noinline bool find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001488 u64 *start, u64 *end, u64 max_bytes,
1489 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001490{
1491 struct rb_node *node;
1492 struct extent_state *state;
1493 u64 cur_start = *start;
Lu Fengqi3522e902018-11-29 11:33:38 +08001494 bool found = false;
Chris Masond1310b22008-01-24 16:13:08 -05001495 u64 total_bytes = 0;
1496
Chris Masoncad321a2008-12-17 14:51:42 -05001497 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001498
Chris Masond1310b22008-01-24 16:13:08 -05001499 /*
1500 * this search will find all the extents that end after
1501 * our range starts.
1502 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001503 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001504 if (!node) {
Lu Fengqi3522e902018-11-29 11:33:38 +08001505 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001506 goto out;
1507 }
1508
Chris Masond3977122009-01-05 21:25:51 -05001509 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001510 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001511 if (found && (state->start != cur_start ||
1512 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001513 goto out;
1514 }
1515 if (!(state->state & EXTENT_DELALLOC)) {
1516 if (!found)
1517 *end = state->end;
1518 goto out;
1519 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001520 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001521 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001522 *cached_state = state;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +02001523 refcount_inc(&state->refs);
Josef Bacikc2a128d2010-02-02 21:19:11 +00001524 }
Lu Fengqi3522e902018-11-29 11:33:38 +08001525 found = true;
Chris Masond1310b22008-01-24 16:13:08 -05001526 *end = state->end;
1527 cur_start = state->end + 1;
1528 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001529 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001530 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001531 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001532 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001533 break;
1534 }
1535out:
Chris Masoncad321a2008-12-17 14:51:42 -05001536 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001537 return found;
1538}
1539
Liu Boda2c7002017-02-10 16:41:05 +01001540static int __process_pages_contig(struct address_space *mapping,
1541 struct page *locked_page,
1542 pgoff_t start_index, pgoff_t end_index,
1543 unsigned long page_ops, pgoff_t *index_ret);
1544
Jeff Mahoney143bede2012-03-01 14:56:26 +01001545static noinline void __unlock_for_delalloc(struct inode *inode,
1546 struct page *locked_page,
1547 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001548{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001549 unsigned long index = start >> PAGE_SHIFT;
1550 unsigned long end_index = end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001551
Liu Bo76c00212017-02-10 16:42:14 +01001552 ASSERT(locked_page);
Chris Masonc8b97812008-10-29 14:49:59 -04001553 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001554 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001555
Liu Bo76c00212017-02-10 16:42:14 +01001556 __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1557 PAGE_UNLOCK, NULL);
Chris Masonc8b97812008-10-29 14:49:59 -04001558}
1559
1560static noinline int lock_delalloc_pages(struct inode *inode,
1561 struct page *locked_page,
1562 u64 delalloc_start,
1563 u64 delalloc_end)
1564{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001565 unsigned long index = delalloc_start >> PAGE_SHIFT;
Liu Bo76c00212017-02-10 16:42:14 +01001566 unsigned long index_ret = index;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001567 unsigned long end_index = delalloc_end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001568 int ret;
Chris Masonc8b97812008-10-29 14:49:59 -04001569
Liu Bo76c00212017-02-10 16:42:14 +01001570 ASSERT(locked_page);
Chris Masonc8b97812008-10-29 14:49:59 -04001571 if (index == locked_page->index && index == end_index)
1572 return 0;
1573
Liu Bo76c00212017-02-10 16:42:14 +01001574 ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1575 end_index, PAGE_LOCK, &index_ret);
1576 if (ret == -EAGAIN)
1577 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1578 (u64)index_ret << PAGE_SHIFT);
Chris Masonc8b97812008-10-29 14:49:59 -04001579 return ret;
1580}
1581
1582/*
Lu Fengqi3522e902018-11-29 11:33:38 +08001583 * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1584 * more than @max_bytes. @Start and @end are used to return the range,
Chris Masonc8b97812008-10-29 14:49:59 -04001585 *
Lu Fengqi3522e902018-11-29 11:33:38 +08001586 * Return: true if we find something
1587 * false if nothing was in the tree
Chris Masonc8b97812008-10-29 14:49:59 -04001588 */
Johannes Thumshirnce9f9672018-11-19 10:38:17 +01001589EXPORT_FOR_TESTS
Lu Fengqi3522e902018-11-29 11:33:38 +08001590noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
Josef Bacik294e30f2013-10-09 12:00:56 -04001591 struct extent_io_tree *tree,
1592 struct page *locked_page, u64 *start,
Nikolay Borisov917aace2018-10-26 14:43:20 +03001593 u64 *end)
Chris Masonc8b97812008-10-29 14:49:59 -04001594{
Nikolay Borisov917aace2018-10-26 14:43:20 +03001595 u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001596 u64 delalloc_start;
1597 u64 delalloc_end;
Lu Fengqi3522e902018-11-29 11:33:38 +08001598 bool found;
Chris Mason9655d292009-09-02 15:22:30 -04001599 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001600 int ret;
1601 int loops = 0;
1602
1603again:
1604 /* step one, find a bunch of delalloc bytes starting at start */
1605 delalloc_start = *start;
1606 delalloc_end = 0;
1607 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001608 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001609 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001610 *start = delalloc_start;
1611 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001612 free_extent_state(cached_state);
Lu Fengqi3522e902018-11-29 11:33:38 +08001613 return false;
Chris Masonc8b97812008-10-29 14:49:59 -04001614 }
1615
1616 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001617 * start comes from the offset of locked_page. We have to lock
1618 * pages in order, so we can't process delalloc bytes before
1619 * locked_page
1620 */
Chris Masond3977122009-01-05 21:25:51 -05001621 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001622 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001623
1624 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001625 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001626 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001627 if (delalloc_end + 1 - delalloc_start > max_bytes)
1628 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001629
Chris Masonc8b97812008-10-29 14:49:59 -04001630 /* step two, lock all the pages after the page that has start */
1631 ret = lock_delalloc_pages(inode, locked_page,
1632 delalloc_start, delalloc_end);
Nikolay Borisov9bfd61d2018-10-26 14:43:21 +03001633 ASSERT(!ret || ret == -EAGAIN);
Chris Masonc8b97812008-10-29 14:49:59 -04001634 if (ret == -EAGAIN) {
1635 /* some of the pages are gone, lets avoid looping by
1636 * shortening the size of the delalloc range we're searching
1637 */
Chris Mason9655d292009-09-02 15:22:30 -04001638 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001639 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001640 if (!loops) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001641 max_bytes = PAGE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001642 loops = 1;
1643 goto again;
1644 } else {
Lu Fengqi3522e902018-11-29 11:33:38 +08001645 found = false;
Chris Masonc8b97812008-10-29 14:49:59 -04001646 goto out_failed;
1647 }
1648 }
Chris Masonc8b97812008-10-29 14:49:59 -04001649
1650 /* step three, lock the state bits for the whole range */
David Sterbaff13db42015-12-03 14:30:40 +01001651 lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001652
1653 /* then test to make sure it is all still delalloc */
1654 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001655 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001656 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001657 unlock_extent_cached(tree, delalloc_start, delalloc_end,
David Sterbae43bbe52017-12-12 21:43:52 +01001658 &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001659 __unlock_for_delalloc(inode, locked_page,
1660 delalloc_start, delalloc_end);
1661 cond_resched();
1662 goto again;
1663 }
Chris Mason9655d292009-09-02 15:22:30 -04001664 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001665 *start = delalloc_start;
1666 *end = delalloc_end;
1667out_failed:
1668 return found;
1669}
1670
Liu Boda2c7002017-02-10 16:41:05 +01001671static int __process_pages_contig(struct address_space *mapping,
1672 struct page *locked_page,
1673 pgoff_t start_index, pgoff_t end_index,
1674 unsigned long page_ops, pgoff_t *index_ret)
Chris Masonc8b97812008-10-29 14:49:59 -04001675{
Liu Bo873695b2017-02-02 17:49:22 -08001676 unsigned long nr_pages = end_index - start_index + 1;
Liu Boda2c7002017-02-10 16:41:05 +01001677 unsigned long pages_locked = 0;
Liu Bo873695b2017-02-02 17:49:22 -08001678 pgoff_t index = start_index;
Chris Masonc8b97812008-10-29 14:49:59 -04001679 struct page *pages[16];
Liu Bo873695b2017-02-02 17:49:22 -08001680 unsigned ret;
Liu Boda2c7002017-02-10 16:41:05 +01001681 int err = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001682 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001683
Liu Boda2c7002017-02-10 16:41:05 +01001684 if (page_ops & PAGE_LOCK) {
1685 ASSERT(page_ops == PAGE_LOCK);
1686 ASSERT(index_ret && *index_ret == start_index);
1687 }
1688
Filipe Manana704de492014-10-06 22:14:22 +01001689 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
Liu Bo873695b2017-02-02 17:49:22 -08001690 mapping_set_error(mapping, -EIO);
Filipe Manana704de492014-10-06 22:14:22 +01001691
Chris Masond3977122009-01-05 21:25:51 -05001692 while (nr_pages > 0) {
Liu Bo873695b2017-02-02 17:49:22 -08001693 ret = find_get_pages_contig(mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001694 min_t(unsigned long,
1695 nr_pages, ARRAY_SIZE(pages)), pages);
Liu Boda2c7002017-02-10 16:41:05 +01001696 if (ret == 0) {
1697 /*
1698 * Only if we're going to lock these pages,
1699 * can we find nothing at @index.
1700 */
1701 ASSERT(page_ops & PAGE_LOCK);
Liu Bo49d4a332017-03-06 18:20:56 -08001702 err = -EAGAIN;
1703 goto out;
Liu Boda2c7002017-02-10 16:41:05 +01001704 }
Chris Mason8b62b722009-09-02 16:53:46 -04001705
Liu Boda2c7002017-02-10 16:41:05 +01001706 for (i = 0; i < ret; i++) {
Josef Bacikc2790a22013-07-29 11:20:47 -04001707 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001708 SetPagePrivate2(pages[i]);
1709
Chris Masonc8b97812008-10-29 14:49:59 -04001710 if (pages[i] == locked_page) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001711 put_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001712 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001713 continue;
1714 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001715 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001716 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001717 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001718 set_page_writeback(pages[i]);
Filipe Manana704de492014-10-06 22:14:22 +01001719 if (page_ops & PAGE_SET_ERROR)
1720 SetPageError(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001721 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001722 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001723 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001724 unlock_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001725 if (page_ops & PAGE_LOCK) {
1726 lock_page(pages[i]);
1727 if (!PageDirty(pages[i]) ||
1728 pages[i]->mapping != mapping) {
1729 unlock_page(pages[i]);
1730 put_page(pages[i]);
1731 err = -EAGAIN;
1732 goto out;
1733 }
1734 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001735 put_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001736 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001737 }
1738 nr_pages -= ret;
1739 index += ret;
1740 cond_resched();
1741 }
Liu Boda2c7002017-02-10 16:41:05 +01001742out:
1743 if (err && index_ret)
1744 *index_ret = start_index + pages_locked - 1;
1745 return err;
Chris Masonc8b97812008-10-29 14:49:59 -04001746}
Chris Masonc8b97812008-10-29 14:49:59 -04001747
Liu Bo873695b2017-02-02 17:49:22 -08001748void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1749 u64 delalloc_end, struct page *locked_page,
1750 unsigned clear_bits,
1751 unsigned long page_ops)
1752{
1753 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
David Sterbaae0f1622017-10-31 16:37:52 +01001754 NULL);
Liu Bo873695b2017-02-02 17:49:22 -08001755
1756 __process_pages_contig(inode->i_mapping, locked_page,
1757 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
Liu Boda2c7002017-02-10 16:41:05 +01001758 page_ops, NULL);
Liu Bo873695b2017-02-02 17:49:22 -08001759}
1760
Chris Masond352ac62008-09-29 15:18:18 -04001761/*
1762 * count the number of bytes in the tree that have a given bit(s)
1763 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1764 * cached. The total number found is returned.
1765 */
Chris Masond1310b22008-01-24 16:13:08 -05001766u64 count_range_bits(struct extent_io_tree *tree,
1767 u64 *start, u64 search_end, u64 max_bytes,
David Sterba9ee49a042015-01-14 19:52:13 +01001768 unsigned bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001769{
1770 struct rb_node *node;
1771 struct extent_state *state;
1772 u64 cur_start = *start;
1773 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001774 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001775 int found = 0;
1776
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301777 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05001778 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05001779
Chris Masoncad321a2008-12-17 14:51:42 -05001780 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001781 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1782 total_bytes = tree->dirty_bytes;
1783 goto out;
1784 }
1785 /*
1786 * this search will find all the extents that end after
1787 * our range starts.
1788 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001789 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001790 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001791 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001792
Chris Masond3977122009-01-05 21:25:51 -05001793 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001794 state = rb_entry(node, struct extent_state, rb_node);
1795 if (state->start > search_end)
1796 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001797 if (contig && found && state->start > last + 1)
1798 break;
1799 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001800 total_bytes += min(search_end, state->end) + 1 -
1801 max(cur_start, state->start);
1802 if (total_bytes >= max_bytes)
1803 break;
1804 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001805 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001806 found = 1;
1807 }
Chris Masonec29ed52011-02-23 16:23:20 -05001808 last = state->end;
1809 } else if (contig && found) {
1810 break;
Chris Masond1310b22008-01-24 16:13:08 -05001811 }
1812 node = rb_next(node);
1813 if (!node)
1814 break;
1815 }
1816out:
Chris Masoncad321a2008-12-17 14:51:42 -05001817 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001818 return total_bytes;
1819}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001820
Chris Masond352ac62008-09-29 15:18:18 -04001821/*
1822 * set the private field for a given byte offset in the tree. If there isn't
1823 * an extent_state there already, this does nothing.
1824 */
Arnd Bergmannf827ba92016-02-22 22:53:20 +01001825static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
David Sterba47dc1962016-02-11 13:24:13 +01001826 struct io_failure_record *failrec)
Chris Masond1310b22008-01-24 16:13:08 -05001827{
1828 struct rb_node *node;
1829 struct extent_state *state;
1830 int ret = 0;
1831
Chris Masoncad321a2008-12-17 14:51:42 -05001832 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001833 /*
1834 * this search will find all the extents that end after
1835 * our range starts.
1836 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001837 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001838 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001839 ret = -ENOENT;
1840 goto out;
1841 }
1842 state = rb_entry(node, struct extent_state, rb_node);
1843 if (state->start != start) {
1844 ret = -ENOENT;
1845 goto out;
1846 }
David Sterba47dc1962016-02-11 13:24:13 +01001847 state->failrec = failrec;
Chris Masond1310b22008-01-24 16:13:08 -05001848out:
Chris Masoncad321a2008-12-17 14:51:42 -05001849 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001850 return ret;
1851}
1852
Arnd Bergmannf827ba92016-02-22 22:53:20 +01001853static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
David Sterba47dc1962016-02-11 13:24:13 +01001854 struct io_failure_record **failrec)
Chris Masond1310b22008-01-24 16:13:08 -05001855{
1856 struct rb_node *node;
1857 struct extent_state *state;
1858 int ret = 0;
1859
Chris Masoncad321a2008-12-17 14:51:42 -05001860 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001861 /*
1862 * this search will find all the extents that end after
1863 * our range starts.
1864 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001865 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001866 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001867 ret = -ENOENT;
1868 goto out;
1869 }
1870 state = rb_entry(node, struct extent_state, rb_node);
1871 if (state->start != start) {
1872 ret = -ENOENT;
1873 goto out;
1874 }
David Sterba47dc1962016-02-11 13:24:13 +01001875 *failrec = state->failrec;
Chris Masond1310b22008-01-24 16:13:08 -05001876out:
Chris Masoncad321a2008-12-17 14:51:42 -05001877 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001878 return ret;
1879}
1880
1881/*
1882 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001883 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001884 * has the bits set. Otherwise, 1 is returned if any bit in the
1885 * range is found set.
1886 */
1887int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001888 unsigned bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001889{
1890 struct extent_state *state = NULL;
1891 struct rb_node *node;
1892 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001893
Chris Masoncad321a2008-12-17 14:51:42 -05001894 spin_lock(&tree->lock);
Filipe Manana27a35072014-07-06 20:09:59 +01001895 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001896 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001897 node = &cached->rb_node;
1898 else
1899 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001900 while (node && start <= end) {
1901 state = rb_entry(node, struct extent_state, rb_node);
1902
1903 if (filled && state->start > start) {
1904 bitset = 0;
1905 break;
1906 }
1907
1908 if (state->start > end)
1909 break;
1910
1911 if (state->state & bits) {
1912 bitset = 1;
1913 if (!filled)
1914 break;
1915 } else if (filled) {
1916 bitset = 0;
1917 break;
1918 }
Chris Mason46562ce2009-09-23 20:23:16 -04001919
1920 if (state->end == (u64)-1)
1921 break;
1922
Chris Masond1310b22008-01-24 16:13:08 -05001923 start = state->end + 1;
1924 if (start > end)
1925 break;
1926 node = rb_next(node);
1927 if (!node) {
1928 if (filled)
1929 bitset = 0;
1930 break;
1931 }
1932 }
Chris Masoncad321a2008-12-17 14:51:42 -05001933 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001934 return bitset;
1935}
Chris Masond1310b22008-01-24 16:13:08 -05001936
1937/*
1938 * helper function to set a given page up to date if all the
1939 * extents in the tree for that page are up to date
1940 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001941static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001942{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001943 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001944 u64 end = start + PAGE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001945 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001946 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001947}
1948
Josef Bacik7870d082017-05-05 11:57:15 -04001949int free_io_failure(struct extent_io_tree *failure_tree,
1950 struct extent_io_tree *io_tree,
1951 struct io_failure_record *rec)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001952{
1953 int ret;
1954 int err = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001955
David Sterba47dc1962016-02-11 13:24:13 +01001956 set_state_failrec(failure_tree, rec->start, NULL);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001957 ret = clear_extent_bits(failure_tree, rec->start,
1958 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02001959 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001960 if (ret)
1961 err = ret;
1962
Josef Bacik7870d082017-05-05 11:57:15 -04001963 ret = clear_extent_bits(io_tree, rec->start,
David Woodhouse53b381b2013-01-29 18:40:14 -05001964 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02001965 EXTENT_DAMAGED);
David Woodhouse53b381b2013-01-29 18:40:14 -05001966 if (ret && !err)
1967 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001968
1969 kfree(rec);
1970 return err;
1971}
1972
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001973/*
1974 * this bypasses the standard btrfs submit functions deliberately, as
1975 * the standard behavior is to write all copies in a raid setup. here we only
1976 * want to write the one bad copy. so we do the mapping for ourselves and issue
1977 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01001978 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001979 * actually prevents the read that triggered the error from finishing.
1980 * currently, there can be no more than two copies of every data bit. thus,
1981 * exactly one rewrite is required.
1982 */
Josef Bacik6ec656b2017-05-05 11:57:14 -04001983int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
1984 u64 length, u64 logical, struct page *page,
1985 unsigned int pg_offset, int mirror_num)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001986{
1987 struct bio *bio;
1988 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001989 u64 map_length = 0;
1990 u64 sector;
1991 struct btrfs_bio *bbio = NULL;
1992 int ret;
1993
Linus Torvalds1751e8a2017-11-27 13:05:09 -08001994 ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001995 BUG_ON(!mirror_num);
1996
David Sterbac5e4c3d2017-06-12 17:29:41 +02001997 bio = btrfs_io_bio_alloc(1);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001998 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001999 map_length = length;
2000
Filipe Mananab5de8d02016-05-27 22:21:27 +01002001 /*
2002 * Avoid races with device replace and make sure our bbio has devices
2003 * associated to its stripes that don't go away while we are doing the
2004 * read repair operation.
2005 */
2006 btrfs_bio_counter_inc_blocked(fs_info);
Nikolay Borisove4ff5fb2017-07-19 10:48:42 +03002007 if (btrfs_is_parity_mirror(fs_info, logical, length)) {
Liu Boc7253282017-03-29 10:53:58 -07002008 /*
2009 * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2010 * to update all raid stripes, but here we just want to correct
2011 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2012 * stripe's dev and sector.
2013 */
2014 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
2015 &map_length, &bbio, 0);
2016 if (ret) {
2017 btrfs_bio_counter_dec(fs_info);
2018 bio_put(bio);
2019 return -EIO;
2020 }
2021 ASSERT(bbio->mirror_num == 1);
2022 } else {
2023 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
2024 &map_length, &bbio, mirror_num);
2025 if (ret) {
2026 btrfs_bio_counter_dec(fs_info);
2027 bio_put(bio);
2028 return -EIO;
2029 }
2030 BUG_ON(mirror_num != bbio->mirror_num);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002031 }
Liu Boc7253282017-03-29 10:53:58 -07002032
2033 sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002034 bio->bi_iter.bi_sector = sector;
Liu Boc7253282017-03-29 10:53:58 -07002035 dev = bbio->stripes[bbio->mirror_num - 1].dev;
Zhao Lei6e9606d2015-01-20 15:11:34 +08002036 btrfs_put_bbio(bbio);
Anand Jainebbede42017-12-04 12:54:52 +08002037 if (!dev || !dev->bdev ||
2038 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
Filipe Mananab5de8d02016-05-27 22:21:27 +01002039 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002040 bio_put(bio);
2041 return -EIO;
2042 }
Christoph Hellwig74d46992017-08-23 19:10:32 +02002043 bio_set_dev(bio, dev->bdev);
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002044 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
Miao Xieffdd2012014-09-12 18:44:00 +08002045 bio_add_page(bio, page, length, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002046
Mike Christie4e49ea42016-06-05 14:31:41 -05002047 if (btrfsic_submit_bio_wait(bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002048 /* try to remap that extent elsewhere? */
Filipe Mananab5de8d02016-05-27 22:21:27 +01002049 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002050 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002051 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002052 return -EIO;
2053 }
2054
David Sterbab14af3b2015-10-08 10:43:10 +02002055 btrfs_info_rl_in_rcu(fs_info,
2056 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
Josef Bacik6ec656b2017-05-05 11:57:14 -04002057 ino, start,
Miao Xie1203b682014-09-12 18:44:01 +08002058 rcu_str_deref(dev->name), sector);
Filipe Mananab5de8d02016-05-27 22:21:27 +01002059 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002060 bio_put(bio);
2061 return 0;
2062}
2063
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002064int repair_eb_io_failure(struct btrfs_fs_info *fs_info,
2065 struct extent_buffer *eb, int mirror_num)
Josef Bacikea466792012-03-26 21:57:36 -04002066{
Josef Bacikea466792012-03-26 21:57:36 -04002067 u64 start = eb->start;
David Sterbacc5e31a2018-03-01 18:20:27 +01002068 int i, num_pages = num_extent_pages(eb);
Chris Masond95603b2012-04-12 15:55:15 -04002069 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002070
David Howellsbc98a422017-07-17 08:45:34 +01002071 if (sb_rdonly(fs_info->sb))
Ilya Dryomov908960c2013-11-03 19:06:39 +02002072 return -EROFS;
2073
Josef Bacikea466792012-03-26 21:57:36 -04002074 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02002075 struct page *p = eb->pages[i];
Miao Xie1203b682014-09-12 18:44:01 +08002076
Josef Bacik6ec656b2017-05-05 11:57:14 -04002077 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
Miao Xie1203b682014-09-12 18:44:01 +08002078 start - page_offset(p), mirror_num);
Josef Bacikea466792012-03-26 21:57:36 -04002079 if (ret)
2080 break;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002081 start += PAGE_SIZE;
Josef Bacikea466792012-03-26 21:57:36 -04002082 }
2083
2084 return ret;
2085}
2086
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002087/*
2088 * each time an IO finishes, we do a fast check in the IO failure tree
2089 * to see if we need to process or clean up an io_failure_record
2090 */
Josef Bacik7870d082017-05-05 11:57:15 -04002091int clean_io_failure(struct btrfs_fs_info *fs_info,
2092 struct extent_io_tree *failure_tree,
2093 struct extent_io_tree *io_tree, u64 start,
2094 struct page *page, u64 ino, unsigned int pg_offset)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002095{
2096 u64 private;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002097 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002098 struct extent_state *state;
2099 int num_copies;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002100 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002101
2102 private = 0;
Josef Bacik7870d082017-05-05 11:57:15 -04002103 ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2104 EXTENT_DIRTY, 0);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002105 if (!ret)
2106 return 0;
2107
Josef Bacik7870d082017-05-05 11:57:15 -04002108 ret = get_state_failrec(failure_tree, start, &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002109 if (ret)
2110 return 0;
2111
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002112 BUG_ON(!failrec->this_mirror);
2113
2114 if (failrec->in_validation) {
2115 /* there was no real error, just free the record */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002116 btrfs_debug(fs_info,
2117 "clean_io_failure: freeing dummy error at %llu",
2118 failrec->start);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002119 goto out;
2120 }
David Howellsbc98a422017-07-17 08:45:34 +01002121 if (sb_rdonly(fs_info->sb))
Ilya Dryomov908960c2013-11-03 19:06:39 +02002122 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002123
Josef Bacik7870d082017-05-05 11:57:15 -04002124 spin_lock(&io_tree->lock);
2125 state = find_first_extent_bit_state(io_tree,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002126 failrec->start,
2127 EXTENT_LOCKED);
Josef Bacik7870d082017-05-05 11:57:15 -04002128 spin_unlock(&io_tree->lock);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002129
Miao Xie883d0de2013-07-25 19:22:35 +08002130 if (state && state->start <= failrec->start &&
2131 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002132 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2133 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002134 if (num_copies > 1) {
Josef Bacik7870d082017-05-05 11:57:15 -04002135 repair_io_failure(fs_info, ino, start, failrec->len,
2136 failrec->logical, page, pg_offset,
2137 failrec->failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002138 }
2139 }
2140
2141out:
Josef Bacik7870d082017-05-05 11:57:15 -04002142 free_io_failure(failure_tree, io_tree, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002143
Miao Xie454ff3d2014-09-12 18:43:58 +08002144 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002145}
2146
Miao Xief6124962014-09-12 18:44:04 +08002147/*
2148 * Can be called when
2149 * - hold extent lock
2150 * - under ordered extent
2151 * - the inode is freeing
2152 */
Nikolay Borisov7ab79562017-02-20 13:50:57 +02002153void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
Miao Xief6124962014-09-12 18:44:04 +08002154{
Nikolay Borisov7ab79562017-02-20 13:50:57 +02002155 struct extent_io_tree *failure_tree = &inode->io_failure_tree;
Miao Xief6124962014-09-12 18:44:04 +08002156 struct io_failure_record *failrec;
2157 struct extent_state *state, *next;
2158
2159 if (RB_EMPTY_ROOT(&failure_tree->state))
2160 return;
2161
2162 spin_lock(&failure_tree->lock);
2163 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2164 while (state) {
2165 if (state->start > end)
2166 break;
2167
2168 ASSERT(state->end <= end);
2169
2170 next = next_state(state);
2171
David Sterba47dc1962016-02-11 13:24:13 +01002172 failrec = state->failrec;
Miao Xief6124962014-09-12 18:44:04 +08002173 free_extent_state(state);
2174 kfree(failrec);
2175
2176 state = next;
2177 }
2178 spin_unlock(&failure_tree->lock);
2179}
2180
Miao Xie2fe63032014-09-12 18:43:59 +08002181int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
David Sterba47dc1962016-02-11 13:24:13 +01002182 struct io_failure_record **failrec_ret)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002183{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002184 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002185 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002186 struct extent_map *em;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002187 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2188 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2189 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002190 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002191 u64 logical;
2192
David Sterba47dc1962016-02-11 13:24:13 +01002193 ret = get_state_failrec(failure_tree, start, &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002194 if (ret) {
2195 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2196 if (!failrec)
2197 return -ENOMEM;
Miao Xie2fe63032014-09-12 18:43:59 +08002198
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002199 failrec->start = start;
2200 failrec->len = end - start + 1;
2201 failrec->this_mirror = 0;
2202 failrec->bio_flags = 0;
2203 failrec->in_validation = 0;
2204
2205 read_lock(&em_tree->lock);
2206 em = lookup_extent_mapping(em_tree, start, failrec->len);
2207 if (!em) {
2208 read_unlock(&em_tree->lock);
2209 kfree(failrec);
2210 return -EIO;
2211 }
2212
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002213 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002214 free_extent_map(em);
2215 em = NULL;
2216 }
2217 read_unlock(&em_tree->lock);
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002218 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002219 kfree(failrec);
2220 return -EIO;
2221 }
Miao Xie2fe63032014-09-12 18:43:59 +08002222
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002223 logical = start - em->start;
2224 logical = em->block_start + logical;
2225 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2226 logical = em->block_start;
2227 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2228 extent_set_compress_type(&failrec->bio_flags,
2229 em->compress_type);
2230 }
Miao Xie2fe63032014-09-12 18:43:59 +08002231
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002232 btrfs_debug(fs_info,
2233 "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2234 logical, start, failrec->len);
Miao Xie2fe63032014-09-12 18:43:59 +08002235
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002236 failrec->logical = logical;
2237 free_extent_map(em);
2238
2239 /* set the bits in the private failure tree */
2240 ret = set_extent_bits(failure_tree, start, end,
David Sterbaceeb0ae2016-04-26 23:54:39 +02002241 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002242 if (ret >= 0)
David Sterba47dc1962016-02-11 13:24:13 +01002243 ret = set_state_failrec(failure_tree, start, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002244 /* set the bits in the inode's tree */
2245 if (ret >= 0)
David Sterbaceeb0ae2016-04-26 23:54:39 +02002246 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002247 if (ret < 0) {
2248 kfree(failrec);
2249 return ret;
2250 }
2251 } else {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002252 btrfs_debug(fs_info,
2253 "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2254 failrec->logical, failrec->start, failrec->len,
2255 failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002256 /*
2257 * when data can be on disk more than twice, add to failrec here
2258 * (e.g. with a list for failed_mirror) to make
2259 * clean_io_failure() clean all those errors at once.
2260 */
2261 }
Miao Xie2fe63032014-09-12 18:43:59 +08002262
2263 *failrec_ret = failrec;
2264
2265 return 0;
2266}
2267
Ming Leia0b60d72017-12-18 20:22:11 +08002268bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
Miao Xie2fe63032014-09-12 18:43:59 +08002269 struct io_failure_record *failrec, int failed_mirror)
2270{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002271 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002272 int num_copies;
2273
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002274 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002275 if (num_copies == 1) {
2276 /*
2277 * we only have a single copy of the data, so don't bother with
2278 * all the retry and error correction code that follows. no
2279 * matter what the error is, it is very likely to persist.
2280 */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002281 btrfs_debug(fs_info,
2282 "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2283 num_copies, failrec->this_mirror, failed_mirror);
Liu Boc3cfb652017-07-13 15:00:50 -07002284 return false;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002285 }
2286
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002287 /*
2288 * there are two premises:
2289 * a) deliver good data to the caller
2290 * b) correct the bad sectors on disk
2291 */
Ming Leia0b60d72017-12-18 20:22:11 +08002292 if (failed_bio_pages > 1) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002293 /*
2294 * to fulfill b), we need to know the exact failing sectors, as
2295 * we don't want to rewrite any more than the failed ones. thus,
2296 * we need separate read requests for the failed bio
2297 *
2298 * if the following BUG_ON triggers, our validation request got
2299 * merged. we need separate requests for our algorithm to work.
2300 */
2301 BUG_ON(failrec->in_validation);
2302 failrec->in_validation = 1;
2303 failrec->this_mirror = failed_mirror;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002304 } else {
2305 /*
2306 * we're ready to fulfill a) and b) alongside. get a good copy
2307 * of the failed sector and if we succeed, we have setup
2308 * everything for repair_io_failure to do the rest for us.
2309 */
2310 if (failrec->in_validation) {
2311 BUG_ON(failrec->this_mirror != failed_mirror);
2312 failrec->in_validation = 0;
2313 failrec->this_mirror = 0;
2314 }
2315 failrec->failed_mirror = failed_mirror;
2316 failrec->this_mirror++;
2317 if (failrec->this_mirror == failed_mirror)
2318 failrec->this_mirror++;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002319 }
2320
Miao Xiefacc8a222013-07-25 19:22:34 +08002321 if (failrec->this_mirror > num_copies) {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002322 btrfs_debug(fs_info,
2323 "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2324 num_copies, failrec->this_mirror, failed_mirror);
Liu Boc3cfb652017-07-13 15:00:50 -07002325 return false;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002326 }
2327
Liu Boc3cfb652017-07-13 15:00:50 -07002328 return true;
Miao Xie2fe63032014-09-12 18:43:59 +08002329}
2330
2331
2332struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2333 struct io_failure_record *failrec,
2334 struct page *page, int pg_offset, int icsum,
Miao Xie8b110e32014-09-12 18:44:03 +08002335 bio_end_io_t *endio_func, void *data)
Miao Xie2fe63032014-09-12 18:43:59 +08002336{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002337 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002338 struct bio *bio;
2339 struct btrfs_io_bio *btrfs_failed_bio;
2340 struct btrfs_io_bio *btrfs_bio;
2341
David Sterbac5e4c3d2017-06-12 17:29:41 +02002342 bio = btrfs_io_bio_alloc(1);
Miao Xie2fe63032014-09-12 18:43:59 +08002343 bio->bi_end_io = endio_func;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002344 bio->bi_iter.bi_sector = failrec->logical >> 9;
Christoph Hellwig74d46992017-08-23 19:10:32 +02002345 bio_set_dev(bio, fs_info->fs_devices->latest_bdev);
Kent Overstreet4f024f32013-10-11 15:44:27 -07002346 bio->bi_iter.bi_size = 0;
Miao Xie8b110e32014-09-12 18:44:03 +08002347 bio->bi_private = data;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002348
Miao Xiefacc8a222013-07-25 19:22:34 +08002349 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2350 if (btrfs_failed_bio->csum) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002351 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2352
2353 btrfs_bio = btrfs_io_bio(bio);
2354 btrfs_bio->csum = btrfs_bio->csum_inline;
Miao Xie2fe63032014-09-12 18:43:59 +08002355 icsum *= csum_size;
2356 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
Miao Xiefacc8a222013-07-25 19:22:34 +08002357 csum_size);
2358 }
2359
Miao Xie2fe63032014-09-12 18:43:59 +08002360 bio_add_page(bio, page, failrec->len, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002361
Miao Xie2fe63032014-09-12 18:43:59 +08002362 return bio;
2363}
2364
2365/*
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002366 * This is a generic handler for readpage errors. If other copies exist, read
2367 * those and write back good data to the failed position. Does not investigate
2368 * in remapping the failed extent elsewhere, hoping the device will be smart
2369 * enough to do this as needed
Miao Xie2fe63032014-09-12 18:43:59 +08002370 */
Miao Xie2fe63032014-09-12 18:43:59 +08002371static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2372 struct page *page, u64 start, u64 end,
2373 int failed_mirror)
2374{
2375 struct io_failure_record *failrec;
2376 struct inode *inode = page->mapping->host;
2377 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Josef Bacik7870d082017-05-05 11:57:15 -04002378 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
Miao Xie2fe63032014-09-12 18:43:59 +08002379 struct bio *bio;
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002380 int read_mode = 0;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002381 blk_status_t status;
Miao Xie2fe63032014-09-12 18:43:59 +08002382 int ret;
Christoph Hellwig8a2ee442019-02-15 19:13:07 +08002383 unsigned failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT;
Miao Xie2fe63032014-09-12 18:43:59 +08002384
Mike Christie1f7ad752016-06-05 14:31:51 -05002385 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
Miao Xie2fe63032014-09-12 18:43:59 +08002386
2387 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2388 if (ret)
2389 return ret;
2390
Ming Leia0b60d72017-12-18 20:22:11 +08002391 if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
Liu Boc3cfb652017-07-13 15:00:50 -07002392 failed_mirror)) {
Josef Bacik7870d082017-05-05 11:57:15 -04002393 free_io_failure(failure_tree, tree, failrec);
Miao Xie2fe63032014-09-12 18:43:59 +08002394 return -EIO;
2395 }
2396
Ming Leia0b60d72017-12-18 20:22:11 +08002397 if (failed_bio_pages > 1)
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002398 read_mode |= REQ_FAILFAST_DEV;
Miao Xie2fe63032014-09-12 18:43:59 +08002399
2400 phy_offset >>= inode->i_sb->s_blocksize_bits;
2401 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2402 start - page_offset(page),
Miao Xie8b110e32014-09-12 18:44:03 +08002403 (int)phy_offset, failed_bio->bi_end_io,
2404 NULL);
David Sterbaebcc3262018-06-29 10:56:53 +02002405 bio->bi_opf = REQ_OP_READ | read_mode;
Miao Xie2fe63032014-09-12 18:43:59 +08002406
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002407 btrfs_debug(btrfs_sb(inode->i_sb),
2408 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2409 read_mode, failrec->this_mirror, failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002410
Linus Torvalds8c27cb32017-07-05 16:41:23 -07002411 status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002412 failrec->bio_flags, 0);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002413 if (status) {
Josef Bacik7870d082017-05-05 11:57:15 -04002414 free_io_failure(failure_tree, tree, failrec);
Miao Xie6c387ab2014-09-12 18:43:57 +08002415 bio_put(bio);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002416 ret = blk_status_to_errno(status);
Miao Xie6c387ab2014-09-12 18:43:57 +08002417 }
2418
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002419 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002420}
2421
Chris Masond1310b22008-01-24 16:13:08 -05002422/* lots and lots of room for performance fixes in the end_bio funcs */
2423
David Sterbab5227c02015-12-03 13:08:59 +01002424void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
Jeff Mahoney87826df2012-02-15 16:23:57 +01002425{
2426 int uptodate = (err == 0);
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002427 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002428
Nikolay Borisovc6297322018-11-08 10:18:08 +02002429 btrfs_writepage_endio_finish_ordered(page, start, end, uptodate);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002430
Jeff Mahoney87826df2012-02-15 16:23:57 +01002431 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002432 ClearPageUptodate(page);
2433 SetPageError(page);
Colin Ian Kingbff5baf2017-05-09 18:14:01 +01002434 ret = err < 0 ? err : -EIO;
Liu Bo5dca6ee2014-05-12 12:47:36 +08002435 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002436 }
Jeff Mahoney87826df2012-02-15 16:23:57 +01002437}
2438
Chris Masond1310b22008-01-24 16:13:08 -05002439/*
2440 * after a writepage IO is done, we need to:
2441 * clear the uptodate bits on error
2442 * clear the writeback bits in the extent tree for this IO
2443 * end_page_writeback if the page has no more pending IO
2444 *
2445 * Scheduling is not allowed, so the extent state tree is expected
2446 * to have one and only one object corresponding to this IO.
2447 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002448static void end_bio_extent_writepage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002449{
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002450 int error = blk_status_to_errno(bio->bi_status);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002451 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002452 u64 start;
2453 u64 end;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002454 int i;
Ming Lei6dc4f102019-02-15 19:13:19 +08002455 struct bvec_iter_all iter_all;
Chris Masond1310b22008-01-24 16:13:08 -05002456
David Sterbac09abff2017-07-13 18:10:07 +02002457 ASSERT(!bio_flagged(bio, BIO_CLONED));
Ming Lei6dc4f102019-02-15 19:13:19 +08002458 bio_for_each_segment_all(bvec, bio, i, iter_all) {
Chris Masond1310b22008-01-24 16:13:08 -05002459 struct page *page = bvec->bv_page;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002460 struct inode *inode = page->mapping->host;
2461 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
David Woodhouse902b22f2008-08-20 08:51:49 -04002462
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002463 /* We always issue full-page reads, but if some block
2464 * in a page fails to read, blk_update_request() will
2465 * advance bv_offset and adjust bv_len to compensate.
2466 * Print a warning for nonzero offsets, and an error
2467 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002468 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2469 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002470 btrfs_err(fs_info,
Frank Holtonefe120a2013-12-20 11:37:06 -05002471 "partial page write in btrfs with offset %u and length %u",
2472 bvec->bv_offset, bvec->bv_len);
2473 else
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002474 btrfs_info(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04002475 "incomplete page write in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002476 bvec->bv_offset, bvec->bv_len);
2477 }
Chris Masond1310b22008-01-24 16:13:08 -05002478
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002479 start = page_offset(page);
2480 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002481
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002482 end_extent_writepage(page, error, start, end);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002483 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002484 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002485
Chris Masond1310b22008-01-24 16:13:08 -05002486 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002487}
2488
Miao Xie883d0de2013-07-25 19:22:35 +08002489static void
2490endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2491 int uptodate)
2492{
2493 struct extent_state *cached = NULL;
2494 u64 end = start + len - 1;
2495
2496 if (uptodate && tree->track_uptodate)
2497 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
David Sterbad810a4b2017-12-07 18:52:54 +01002498 unlock_extent_cached_atomic(tree, start, end, &cached);
Miao Xie883d0de2013-07-25 19:22:35 +08002499}
2500
Chris Masond1310b22008-01-24 16:13:08 -05002501/*
2502 * after a readpage IO is done, we need to:
2503 * clear the uptodate bits on error
2504 * set the uptodate bits if things worked
2505 * set the page up to date if all extents in the tree are uptodate
2506 * clear the lock bit in the extent tree
2507 * unlock the page if there are no other extents locked for it
2508 *
2509 * Scheduling is not allowed, so the extent state tree is expected
2510 * to have one and only one object corresponding to this IO.
2511 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002512static void end_bio_extent_readpage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002513{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002514 struct bio_vec *bvec;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002515 int uptodate = !bio->bi_status;
Miao Xiefacc8a222013-07-25 19:22:34 +08002516 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
Josef Bacik7870d082017-05-05 11:57:15 -04002517 struct extent_io_tree *tree, *failure_tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002518 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002519 u64 start;
2520 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002521 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002522 u64 extent_start = 0;
2523 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002524 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002525 int ret;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002526 int i;
Ming Lei6dc4f102019-02-15 19:13:19 +08002527 struct bvec_iter_all iter_all;
Chris Masond1310b22008-01-24 16:13:08 -05002528
David Sterbac09abff2017-07-13 18:10:07 +02002529 ASSERT(!bio_flagged(bio, BIO_CLONED));
Ming Lei6dc4f102019-02-15 19:13:19 +08002530 bio_for_each_segment_all(bvec, bio, i, iter_all) {
Chris Masond1310b22008-01-24 16:13:08 -05002531 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002532 struct inode *inode = page->mapping->host;
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002533 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002534 bool data_inode = btrfs_ino(BTRFS_I(inode))
2535 != BTRFS_BTREE_INODE_OBJECTID;
Arne Jansen507903b2011-04-06 10:02:20 +00002536
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002537 btrfs_debug(fs_info,
2538 "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002539 (u64)bio->bi_iter.bi_sector, bio->bi_status,
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002540 io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002541 tree = &BTRFS_I(inode)->io_tree;
Josef Bacik7870d082017-05-05 11:57:15 -04002542 failure_tree = &BTRFS_I(inode)->io_failure_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002543
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002544 /* We always issue full-page reads, but if some block
2545 * in a page fails to read, blk_update_request() will
2546 * advance bv_offset and adjust bv_len to compensate.
2547 * Print a warning for nonzero offsets, and an error
2548 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002549 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2550 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002551 btrfs_err(fs_info,
2552 "partial page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002553 bvec->bv_offset, bvec->bv_len);
2554 else
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002555 btrfs_info(fs_info,
2556 "incomplete page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002557 bvec->bv_offset, bvec->bv_len);
2558 }
Chris Masond1310b22008-01-24 16:13:08 -05002559
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002560 start = page_offset(page);
2561 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002562 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002563
Chris Mason9be33952013-05-17 18:30:14 -04002564 mirror = io_bio->mirror_num;
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002565 if (likely(uptodate)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002566 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2567 page, start, end,
2568 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002569 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002570 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002571 else
Josef Bacik7870d082017-05-05 11:57:15 -04002572 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2573 failure_tree, tree, start,
2574 page,
2575 btrfs_ino(BTRFS_I(inode)), 0);
Chris Masond1310b22008-01-24 16:13:08 -05002576 }
Josef Bacikea466792012-03-26 21:57:36 -04002577
Miao Xief2a09da2013-07-25 19:22:33 +08002578 if (likely(uptodate))
2579 goto readpage_ok;
2580
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002581 if (data_inode) {
Liu Bo9d0d1c82017-03-24 15:04:50 -07002582
2583 /*
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002584 * The generic bio_readpage_error handles errors the
2585 * following way: If possible, new read requests are
2586 * created and submitted and will end up in
2587 * end_bio_extent_readpage as well (if we're lucky,
2588 * not in the !uptodate case). In that case it returns
2589 * 0 and we just go on with the next page in our bio.
2590 * If it can't handle the error it will return -EIO and
2591 * we remain responsible for that page.
Liu Bo9d0d1c82017-03-24 15:04:50 -07002592 */
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002593 ret = bio_readpage_error(bio, offset, page, start, end,
2594 mirror);
2595 if (ret == 0) {
2596 uptodate = !bio->bi_status;
2597 offset += len;
2598 continue;
2599 }
2600 } else {
2601 struct extent_buffer *eb;
2602
2603 eb = (struct extent_buffer *)page->private;
2604 set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
2605 eb->read_mirror = mirror;
2606 atomic_dec(&eb->io_pages);
2607 if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD,
2608 &eb->bflags))
2609 btree_readahead_hook(eb, -EIO);
2610
2611 ret = -EIO;
Chris Mason7e383262008-04-09 16:28:12 -04002612 }
Miao Xief2a09da2013-07-25 19:22:33 +08002613readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002614 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002615 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002616 pgoff_t end_index = i_size >> PAGE_SHIFT;
Liu Boa583c022014-08-19 23:32:22 +08002617 unsigned off;
Josef Bacika71754f2013-06-17 17:14:39 -04002618
2619 /* Zero out the end if this page straddles i_size */
Johannes Thumshirn70730172018-12-05 15:23:03 +01002620 off = offset_in_page(i_size);
Liu Boa583c022014-08-19 23:32:22 +08002621 if (page->index == end_index && off)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002622 zero_user_segment(page, off, PAGE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002623 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002624 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002625 ClearPageUptodate(page);
2626 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002627 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002628 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002629 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002630
2631 if (unlikely(!uptodate)) {
2632 if (extent_len) {
2633 endio_readpage_release_extent(tree,
2634 extent_start,
2635 extent_len, 1);
2636 extent_start = 0;
2637 extent_len = 0;
2638 }
2639 endio_readpage_release_extent(tree, start,
2640 end - start + 1, 0);
2641 } else if (!extent_len) {
2642 extent_start = start;
2643 extent_len = end + 1 - start;
2644 } else if (extent_start + extent_len == start) {
2645 extent_len += end + 1 - start;
2646 } else {
2647 endio_readpage_release_extent(tree, extent_start,
2648 extent_len, uptodate);
2649 extent_start = start;
2650 extent_len = end + 1 - start;
2651 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002652 }
Chris Masond1310b22008-01-24 16:13:08 -05002653
Miao Xie883d0de2013-07-25 19:22:35 +08002654 if (extent_len)
2655 endio_readpage_release_extent(tree, extent_start, extent_len,
2656 uptodate);
David Sterbab3a0dd52018-11-22 17:16:49 +01002657 btrfs_io_bio_free_csum(io_bio);
Chris Masond1310b22008-01-24 16:13:08 -05002658 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002659}
2660
Chris Mason9be33952013-05-17 18:30:14 -04002661/*
David Sterba184f9992017-06-12 17:29:39 +02002662 * Initialize the members up to but not including 'bio'. Use after allocating a
2663 * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2664 * 'bio' because use of __GFP_ZERO is not supported.
Chris Mason9be33952013-05-17 18:30:14 -04002665 */
David Sterba184f9992017-06-12 17:29:39 +02002666static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
Chris Masond1310b22008-01-24 16:13:08 -05002667{
David Sterba184f9992017-06-12 17:29:39 +02002668 memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2669}
2670
2671/*
David Sterba6e707bc2017-06-02 17:26:26 +02002672 * The following helpers allocate a bio. As it's backed by a bioset, it'll
2673 * never fail. We're returning a bio right now but you can call btrfs_io_bio
2674 * for the appropriate container_of magic
Chris Masond1310b22008-01-24 16:13:08 -05002675 */
David Sterbac821e7f32017-06-02 18:35:36 +02002676struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte)
Chris Masond1310b22008-01-24 16:13:08 -05002677{
2678 struct bio *bio;
2679
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002680 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
Christoph Hellwig74d46992017-08-23 19:10:32 +02002681 bio_set_dev(bio, bdev);
David Sterbac821e7f32017-06-02 18:35:36 +02002682 bio->bi_iter.bi_sector = first_byte >> 9;
David Sterba184f9992017-06-12 17:29:39 +02002683 btrfs_io_bio_init(btrfs_io_bio(bio));
Chris Masond1310b22008-01-24 16:13:08 -05002684 return bio;
2685}
2686
David Sterba8b6c1d52017-06-02 17:48:13 +02002687struct bio *btrfs_bio_clone(struct bio *bio)
Chris Mason9be33952013-05-17 18:30:14 -04002688{
Miao Xie23ea8e52014-09-12 18:43:54 +08002689 struct btrfs_io_bio *btrfs_bio;
2690 struct bio *new;
Chris Mason9be33952013-05-17 18:30:14 -04002691
David Sterba6e707bc2017-06-02 17:26:26 +02002692 /* Bio allocation backed by a bioset does not fail */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002693 new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
David Sterba6e707bc2017-06-02 17:26:26 +02002694 btrfs_bio = btrfs_io_bio(new);
David Sterba184f9992017-06-12 17:29:39 +02002695 btrfs_io_bio_init(btrfs_bio);
David Sterba6e707bc2017-06-02 17:26:26 +02002696 btrfs_bio->iter = bio->bi_iter;
Miao Xie23ea8e52014-09-12 18:43:54 +08002697 return new;
2698}
Chris Mason9be33952013-05-17 18:30:14 -04002699
David Sterbac5e4c3d2017-06-12 17:29:41 +02002700struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
Chris Mason9be33952013-05-17 18:30:14 -04002701{
Miao Xiefacc8a222013-07-25 19:22:34 +08002702 struct bio *bio;
2703
David Sterba6e707bc2017-06-02 17:26:26 +02002704 /* Bio allocation backed by a bioset does not fail */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002705 bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
David Sterba184f9992017-06-12 17:29:39 +02002706 btrfs_io_bio_init(btrfs_io_bio(bio));
Miao Xiefacc8a222013-07-25 19:22:34 +08002707 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002708}
2709
Liu Boe4770942017-05-16 10:57:14 -07002710struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
Liu Bo2f8e9142017-05-15 17:43:31 -07002711{
2712 struct bio *bio;
2713 struct btrfs_io_bio *btrfs_bio;
2714
2715 /* this will never fail when it's backed by a bioset */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002716 bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
Liu Bo2f8e9142017-05-15 17:43:31 -07002717 ASSERT(bio);
2718
2719 btrfs_bio = btrfs_io_bio(bio);
David Sterba184f9992017-06-12 17:29:39 +02002720 btrfs_io_bio_init(btrfs_bio);
Liu Bo2f8e9142017-05-15 17:43:31 -07002721
2722 bio_trim(bio, offset >> 9, size >> 9);
Liu Bo17347ce2017-05-15 15:33:27 -07002723 btrfs_bio->iter = bio->bi_iter;
Liu Bo2f8e9142017-05-15 17:43:31 -07002724 return bio;
2725}
Chris Mason9be33952013-05-17 18:30:14 -04002726
David Sterba4b81ba42017-06-06 19:14:26 +02002727/*
2728 * @opf: bio REQ_OP_* and REQ_* flags as one value
David Sterbab8b3d622017-06-12 19:50:41 +02002729 * @tree: tree so we can call our merge_bio hook
2730 * @wbc: optional writeback control for io accounting
2731 * @page: page to add to the bio
2732 * @pg_offset: offset of the new bio or to check whether we are adding
2733 * a contiguous page to the previous one
2734 * @size: portion of page that we want to write
2735 * @offset: starting offset in the page
2736 * @bdev: attach newly created bios to this bdev
David Sterba5c2b1fd2017-06-06 19:22:55 +02002737 * @bio_ret: must be valid pointer, newly allocated bio will be stored there
David Sterbab8b3d622017-06-12 19:50:41 +02002738 * @end_io_func: end_io callback for new bio
2739 * @mirror_num: desired mirror to read/write
2740 * @prev_bio_flags: flags of previous bio to see if we can merge the current one
2741 * @bio_flags: flags of the current bio to see if we can merge them
David Sterba4b81ba42017-06-06 19:14:26 +02002742 */
2743static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
Chris Masonda2f0f72015-07-02 13:57:22 -07002744 struct writeback_control *wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02002745 struct page *page, u64 offset,
David Sterba6c5a4e22017-10-04 17:10:34 +02002746 size_t size, unsigned long pg_offset,
Chris Masond1310b22008-01-24 16:13:08 -05002747 struct block_device *bdev,
2748 struct bio **bio_ret,
Chris Masonf1885912008-04-09 16:28:12 -04002749 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002750 int mirror_num,
2751 unsigned long prev_bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002752 unsigned long bio_flags,
2753 bool force_bio_submit)
Chris Masond1310b22008-01-24 16:13:08 -05002754{
2755 int ret = 0;
2756 struct bio *bio;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002757 size_t page_size = min_t(size_t, size, PAGE_SIZE);
David Sterba6273b7f2017-10-04 17:30:11 +02002758 sector_t sector = offset >> 9;
Chris Masond1310b22008-01-24 16:13:08 -05002759
David Sterba5c2b1fd2017-06-06 19:22:55 +02002760 ASSERT(bio_ret);
2761
2762 if (*bio_ret) {
David Sterba0c8508a2017-06-12 20:00:43 +02002763 bool contig;
2764 bool can_merge = true;
2765
Chris Masond1310b22008-01-24 16:13:08 -05002766 bio = *bio_ret;
David Sterba0c8508a2017-06-12 20:00:43 +02002767 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
Kent Overstreet4f024f32013-10-11 15:44:27 -07002768 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002769 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002770 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002771
Nikolay Borisovda12fe52018-11-27 20:57:58 +02002772 ASSERT(tree->ops);
2773 if (btrfs_bio_fits_in_stripe(page, page_size, bio, bio_flags))
David Sterba0c8508a2017-06-12 20:00:43 +02002774 can_merge = false;
2775
2776 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
Filipe Manana005efed2015-09-14 09:09:31 +01002777 force_bio_submit ||
David Sterba6c5a4e22017-10-04 17:10:34 +02002778 bio_add_page(bio, page, page_size, pg_offset) < page_size) {
Mike Christie1f7ad752016-06-05 14:31:51 -05002779 ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
Naohiro Aota289454a2015-01-06 01:01:03 +09002780 if (ret < 0) {
2781 *bio_ret = NULL;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002782 return ret;
Naohiro Aota289454a2015-01-06 01:01:03 +09002783 }
Chris Masond1310b22008-01-24 16:13:08 -05002784 bio = NULL;
2785 } else {
Chris Masonda2f0f72015-07-02 13:57:22 -07002786 if (wbc)
2787 wbc_account_io(wbc, page, page_size);
Chris Masond1310b22008-01-24 16:13:08 -05002788 return 0;
2789 }
2790 }
Chris Masonc8b97812008-10-29 14:49:59 -04002791
David Sterba6273b7f2017-10-04 17:30:11 +02002792 bio = btrfs_bio_alloc(bdev, offset);
David Sterba6c5a4e22017-10-04 17:10:34 +02002793 bio_add_page(bio, page, page_size, pg_offset);
Chris Masond1310b22008-01-24 16:13:08 -05002794 bio->bi_end_io = end_io_func;
2795 bio->bi_private = tree;
Jens Axboee6959b92017-06-27 11:51:28 -06002796 bio->bi_write_hint = page->mapping->host->i_write_hint;
David Sterba4b81ba42017-06-06 19:14:26 +02002797 bio->bi_opf = opf;
Chris Masonda2f0f72015-07-02 13:57:22 -07002798 if (wbc) {
2799 wbc_init_bio(wbc, bio);
2800 wbc_account_io(wbc, page, page_size);
2801 }
Chris Mason70dec802008-01-29 09:59:12 -05002802
David Sterba5c2b1fd2017-06-06 19:22:55 +02002803 *bio_ret = bio;
Chris Masond1310b22008-01-24 16:13:08 -05002804
2805 return ret;
2806}
2807
Eric Sandeen48a3b632013-04-25 20:41:01 +00002808static void attach_extent_buffer_page(struct extent_buffer *eb,
2809 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002810{
2811 if (!PagePrivate(page)) {
2812 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002813 get_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002814 set_page_private(page, (unsigned long)eb);
2815 } else {
2816 WARN_ON(page->private != (unsigned long)eb);
2817 }
2818}
2819
Chris Masond1310b22008-01-24 16:13:08 -05002820void set_page_extent_mapped(struct page *page)
2821{
2822 if (!PagePrivate(page)) {
2823 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002824 get_page(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002825 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002826 }
2827}
2828
Miao Xie125bac012013-07-25 19:22:37 +08002829static struct extent_map *
2830__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2831 u64 start, u64 len, get_extent_t *get_extent,
2832 struct extent_map **em_cached)
2833{
2834 struct extent_map *em;
2835
2836 if (em_cached && *em_cached) {
2837 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00002838 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08002839 start < extent_map_end(em)) {
Elena Reshetova490b54d2017-03-03 10:55:12 +02002840 refcount_inc(&em->refs);
Miao Xie125bac012013-07-25 19:22:37 +08002841 return em;
2842 }
2843
2844 free_extent_map(em);
2845 *em_cached = NULL;
2846 }
2847
Nikolay Borisovfc4f21b12017-02-20 13:51:06 +02002848 em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
Miao Xie125bac012013-07-25 19:22:37 +08002849 if (em_cached && !IS_ERR_OR_NULL(em)) {
2850 BUG_ON(*em_cached);
Elena Reshetova490b54d2017-03-03 10:55:12 +02002851 refcount_inc(&em->refs);
Miao Xie125bac012013-07-25 19:22:37 +08002852 *em_cached = em;
2853 }
2854 return em;
2855}
Chris Masond1310b22008-01-24 16:13:08 -05002856/*
2857 * basic readpage implementation. Locked extent state structs are inserted
2858 * into the tree that are removed when the IO is done (by the end_io
2859 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002860 * XXX JDM: This needs looking at to ensure proper page locking
Liu Bobaf863b2016-07-11 10:39:07 -07002861 * return 0 on success, otherwise return error
Chris Masond1310b22008-01-24 16:13:08 -05002862 */
Miao Xie99740902013-07-25 19:22:36 +08002863static int __do_readpage(struct extent_io_tree *tree,
2864 struct page *page,
2865 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002866 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002867 struct bio **bio, int mirror_num,
David Sterbaf1c77c52017-06-06 19:03:49 +02002868 unsigned long *bio_flags, unsigned int read_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002869 u64 *prev_em_start)
Chris Masond1310b22008-01-24 16:13:08 -05002870{
2871 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002872 u64 start = page_offset(page);
David Sterba8eec8292017-06-06 19:50:13 +02002873 const u64 end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002874 u64 cur = start;
2875 u64 extent_offset;
2876 u64 last_byte = i_size_read(inode);
2877 u64 block_start;
2878 u64 cur_end;
Chris Masond1310b22008-01-24 16:13:08 -05002879 struct extent_map *em;
2880 struct block_device *bdev;
Liu Bobaf863b2016-07-11 10:39:07 -07002881 int ret = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002882 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02002883 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002884 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002885 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002886 size_t blocksize = inode->i_sb->s_blocksize;
Filipe Manana7f042a82016-01-27 19:17:20 +00002887 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002888
2889 set_page_extent_mapped(page);
2890
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002891 if (!PageUptodate(page)) {
2892 if (cleancache_get_page(page) == 0) {
2893 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08002894 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002895 goto out;
2896 }
2897 }
2898
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002899 if (page->index == last_byte >> PAGE_SHIFT) {
Chris Masonc8b97812008-10-29 14:49:59 -04002900 char *userpage;
Johannes Thumshirn70730172018-12-05 15:23:03 +01002901 size_t zero_offset = offset_in_page(last_byte);
Chris Masonc8b97812008-10-29 14:49:59 -04002902
2903 if (zero_offset) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002904 iosize = PAGE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002905 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002906 memset(userpage + zero_offset, 0, iosize);
2907 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002908 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002909 }
2910 }
Chris Masond1310b22008-01-24 16:13:08 -05002911 while (cur <= end) {
Filipe Manana005efed2015-09-14 09:09:31 +01002912 bool force_bio_submit = false;
David Sterba6273b7f2017-10-04 17:30:11 +02002913 u64 offset;
Josef Bacikc8f2f242013-02-11 11:33:00 -05002914
Chris Masond1310b22008-01-24 16:13:08 -05002915 if (cur >= last_byte) {
2916 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002917 struct extent_state *cached = NULL;
2918
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002919 iosize = PAGE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002920 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002921 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002922 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002923 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002924 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002925 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00002926 unlock_extent_cached(tree, cur,
David Sterbae43bbe52017-12-12 21:43:52 +01002927 cur + iosize - 1, &cached);
Chris Masond1310b22008-01-24 16:13:08 -05002928 break;
2929 }
Miao Xie125bac012013-07-25 19:22:37 +08002930 em = __get_extent_map(inode, page, pg_offset, cur,
2931 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02002932 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002933 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00002934 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002935 break;
2936 }
Chris Masond1310b22008-01-24 16:13:08 -05002937 extent_offset = cur - em->start;
2938 BUG_ON(extent_map_end(em) <= cur);
2939 BUG_ON(end < cur);
2940
Li Zefan261507a02010-12-17 14:21:50 +08002941 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07002942 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002943 extent_set_compress_type(&this_bio_flag,
2944 em->compress_type);
2945 }
Chris Masonc8b97812008-10-29 14:49:59 -04002946
Chris Masond1310b22008-01-24 16:13:08 -05002947 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2948 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00002949 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04002950 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2951 disk_io_size = em->block_len;
David Sterba6273b7f2017-10-04 17:30:11 +02002952 offset = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04002953 } else {
David Sterba6273b7f2017-10-04 17:30:11 +02002954 offset = em->block_start + extent_offset;
Chris Masonc8b97812008-10-29 14:49:59 -04002955 disk_io_size = iosize;
2956 }
Chris Masond1310b22008-01-24 16:13:08 -05002957 bdev = em->bdev;
2958 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002959 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2960 block_start = EXTENT_MAP_HOLE;
Filipe Manana005efed2015-09-14 09:09:31 +01002961
2962 /*
2963 * If we have a file range that points to a compressed extent
2964 * and it's followed by a consecutive file range that points to
2965 * to the same compressed extent (possibly with a different
2966 * offset and/or length, so it either points to the whole extent
2967 * or only part of it), we must make sure we do not submit a
2968 * single bio to populate the pages for the 2 ranges because
2969 * this makes the compressed extent read zero out the pages
2970 * belonging to the 2nd range. Imagine the following scenario:
2971 *
2972 * File layout
2973 * [0 - 8K] [8K - 24K]
2974 * | |
2975 * | |
2976 * points to extent X, points to extent X,
2977 * offset 4K, length of 8K offset 0, length 16K
2978 *
2979 * [extent X, compressed length = 4K uncompressed length = 16K]
2980 *
2981 * If the bio to read the compressed extent covers both ranges,
2982 * it will decompress extent X into the pages belonging to the
2983 * first range and then it will stop, zeroing out the remaining
2984 * pages that belong to the other range that points to extent X.
2985 * So here we make sure we submit 2 bios, one for the first
2986 * range and another one for the third range. Both will target
2987 * the same physical extent from disk, but we can't currently
2988 * make the compressed bio endio callback populate the pages
2989 * for both ranges because each compressed bio is tightly
2990 * coupled with a single extent map, and each range can have
2991 * an extent map with a different offset value relative to the
2992 * uncompressed data of our extent and different lengths. This
2993 * is a corner case so we prioritize correctness over
2994 * non-optimal behavior (submitting 2 bios for the same extent).
2995 */
2996 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
2997 prev_em_start && *prev_em_start != (u64)-1 &&
Filipe Manana8e928212019-02-14 15:17:20 +00002998 *prev_em_start != em->start)
Filipe Manana005efed2015-09-14 09:09:31 +01002999 force_bio_submit = true;
3000
3001 if (prev_em_start)
Filipe Manana8e928212019-02-14 15:17:20 +00003002 *prev_em_start = em->start;
Filipe Manana005efed2015-09-14 09:09:31 +01003003
Chris Masond1310b22008-01-24 16:13:08 -05003004 free_extent_map(em);
3005 em = NULL;
3006
3007 /* we've found a hole, just zero and go on */
3008 if (block_start == EXTENT_MAP_HOLE) {
3009 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003010 struct extent_state *cached = NULL;
3011
Cong Wang7ac687d2011-11-25 23:14:28 +08003012 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003013 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003014 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003015 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003016
3017 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003018 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00003019 unlock_extent_cached(tree, cur,
David Sterbae43bbe52017-12-12 21:43:52 +01003020 cur + iosize - 1, &cached);
Chris Masond1310b22008-01-24 16:13:08 -05003021 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003022 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003023 continue;
3024 }
3025 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04003026 if (test_range_bit(tree, cur, cur_end,
3027 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04003028 check_page_uptodate(tree, page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003029 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003030 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003031 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003032 continue;
3033 }
Chris Mason70dec802008-01-29 09:59:12 -05003034 /* we have an inline extent but it didn't get marked up
3035 * to date. Error out
3036 */
3037 if (block_start == EXTENT_MAP_INLINE) {
3038 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003039 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05003040 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003041 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05003042 continue;
3043 }
Chris Masond1310b22008-01-24 16:13:08 -05003044
David Sterba4b81ba42017-06-06 19:14:26 +02003045 ret = submit_extent_page(REQ_OP_READ | read_flags, tree, NULL,
David Sterba6273b7f2017-10-04 17:30:11 +02003046 page, offset, disk_io_size,
3047 pg_offset, bdev, bio,
Chris Masonc8b97812008-10-29 14:49:59 -04003048 end_bio_extent_readpage, mirror_num,
3049 *bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01003050 this_bio_flag,
3051 force_bio_submit);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003052 if (!ret) {
3053 nr++;
3054 *bio_flags = this_bio_flag;
3055 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003056 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003057 unlock_extent(tree, cur, cur + iosize - 1);
Liu Bobaf863b2016-07-11 10:39:07 -07003058 goto out;
Josef Bacikedd33c92012-10-05 16:40:32 -04003059 }
Chris Masond1310b22008-01-24 16:13:08 -05003060 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003061 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003062 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003063out:
Chris Masond1310b22008-01-24 16:13:08 -05003064 if (!nr) {
3065 if (!PageError(page))
3066 SetPageUptodate(page);
3067 unlock_page(page);
3068 }
Liu Bobaf863b2016-07-11 10:39:07 -07003069 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05003070}
3071
Miao Xie99740902013-07-25 19:22:36 +08003072static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3073 struct page *pages[], int nr_pages,
3074 u64 start, u64 end,
Miao Xie125bac012013-07-25 19:22:37 +08003075 struct extent_map **em_cached,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003076 struct bio **bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003077 unsigned long *bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003078 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003079{
3080 struct inode *inode;
3081 struct btrfs_ordered_extent *ordered;
3082 int index;
3083
3084 inode = pages[0]->mapping->host;
3085 while (1) {
3086 lock_extent(tree, start, end);
Nikolay Borisova776c6f2017-02-20 13:50:49 +02003087 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
Miao Xie99740902013-07-25 19:22:36 +08003088 end - start + 1);
3089 if (!ordered)
3090 break;
3091 unlock_extent(tree, start, end);
3092 btrfs_start_ordered_extent(inode, ordered, 1);
3093 btrfs_put_ordered_extent(ordered);
3094 }
3095
3096 for (index = 0; index < nr_pages; index++) {
David Sterba4ef77692017-06-23 04:09:57 +02003097 __do_readpage(tree, pages[index], btrfs_get_extent, em_cached,
Jens Axboe5e9d3982018-08-17 15:45:39 -07003098 bio, 0, bio_flags, REQ_RAHEAD, prev_em_start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003099 put_page(pages[index]);
Miao Xie99740902013-07-25 19:22:36 +08003100 }
3101}
3102
3103static void __extent_readpages(struct extent_io_tree *tree,
3104 struct page *pages[],
David Sterbae4d17ef2017-06-23 04:09:57 +02003105 int nr_pages,
Miao Xie125bac012013-07-25 19:22:37 +08003106 struct extent_map **em_cached,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003107 struct bio **bio, unsigned long *bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003108 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003109{
Stefan Behrens35a36212013-08-14 18:12:25 +02003110 u64 start = 0;
Miao Xie99740902013-07-25 19:22:36 +08003111 u64 end = 0;
3112 u64 page_start;
3113 int index;
Stefan Behrens35a36212013-08-14 18:12:25 +02003114 int first_index = 0;
Miao Xie99740902013-07-25 19:22:36 +08003115
3116 for (index = 0; index < nr_pages; index++) {
3117 page_start = page_offset(pages[index]);
3118 if (!end) {
3119 start = page_start;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003120 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003121 first_index = index;
3122 } else if (end + 1 == page_start) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003123 end += PAGE_SIZE;
Miao Xie99740902013-07-25 19:22:36 +08003124 } else {
3125 __do_contiguous_readpages(tree, &pages[first_index],
3126 index - first_index, start,
David Sterba4ef77692017-06-23 04:09:57 +02003127 end, em_cached,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003128 bio, bio_flags,
Mike Christie1f7ad752016-06-05 14:31:51 -05003129 prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003130 start = page_start;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003131 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003132 first_index = index;
3133 }
3134 }
3135
3136 if (end)
3137 __do_contiguous_readpages(tree, &pages[first_index],
3138 index - first_index, start,
David Sterba4ef77692017-06-23 04:09:57 +02003139 end, em_cached, bio,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003140 bio_flags, prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003141}
3142
3143static int __extent_read_full_page(struct extent_io_tree *tree,
3144 struct page *page,
3145 get_extent_t *get_extent,
3146 struct bio **bio, int mirror_num,
David Sterbaf1c77c52017-06-06 19:03:49 +02003147 unsigned long *bio_flags,
3148 unsigned int read_flags)
Miao Xie99740902013-07-25 19:22:36 +08003149{
3150 struct inode *inode = page->mapping->host;
3151 struct btrfs_ordered_extent *ordered;
3152 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003153 u64 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003154 int ret;
3155
3156 while (1) {
3157 lock_extent(tree, start, end);
Nikolay Borisova776c6f2017-02-20 13:50:49 +02003158 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003159 PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08003160 if (!ordered)
3161 break;
3162 unlock_extent(tree, start, end);
3163 btrfs_start_ordered_extent(inode, ordered, 1);
3164 btrfs_put_ordered_extent(ordered);
3165 }
3166
Miao Xie125bac012013-07-25 19:22:37 +08003167 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003168 bio_flags, read_flags, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003169 return ret;
3170}
3171
Chris Masond1310b22008-01-24 16:13:08 -05003172int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003173 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003174{
3175 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003176 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003177 int ret;
3178
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003179 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003180 &bio_flags, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003181 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05003182 ret = submit_one_bio(bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003183 return ret;
3184}
Chris Masond1310b22008-01-24 16:13:08 -05003185
David Sterba3d4b9492017-02-10 19:33:41 +01003186static void update_nr_written(struct writeback_control *wbc,
Liu Boa91326672016-03-07 16:56:21 -08003187 unsigned long nr_written)
Chris Mason11c83492009-04-20 15:50:09 -04003188{
3189 wbc->nr_to_write -= nr_written;
Chris Mason11c83492009-04-20 15:50:09 -04003190}
3191
Chris Masond1310b22008-01-24 16:13:08 -05003192/*
Chris Mason40f76582014-05-21 13:35:51 -07003193 * helper for __extent_writepage, doing all of the delayed allocation setup.
3194 *
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003195 * This returns 1 if btrfs_run_delalloc_range function did all the work required
Chris Mason40f76582014-05-21 13:35:51 -07003196 * to write the page (copy into inline extent). In this case the IO has
3197 * been started and the page is already unlocked.
3198 *
3199 * This returns 0 if all went well (page still locked)
3200 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003201 */
Chris Mason40f76582014-05-21 13:35:51 -07003202static noinline_for_stack int writepage_delalloc(struct inode *inode,
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003203 struct page *page, struct writeback_control *wbc,
3204 u64 delalloc_start, unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003205{
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003206 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003207 u64 page_end = delalloc_start + PAGE_SIZE - 1;
Lu Fengqi3522e902018-11-29 11:33:38 +08003208 bool found;
Chris Mason40f76582014-05-21 13:35:51 -07003209 u64 delalloc_to_write = 0;
3210 u64 delalloc_end = 0;
3211 int ret;
3212 int page_started = 0;
3213
Chris Mason40f76582014-05-21 13:35:51 -07003214
3215 while (delalloc_end < page_end) {
Lu Fengqi3522e902018-11-29 11:33:38 +08003216 found = find_lock_delalloc_range(inode, tree,
Chris Mason40f76582014-05-21 13:35:51 -07003217 page,
3218 &delalloc_start,
Nikolay Borisov917aace2018-10-26 14:43:20 +03003219 &delalloc_end);
Lu Fengqi3522e902018-11-29 11:33:38 +08003220 if (!found) {
Chris Mason40f76582014-05-21 13:35:51 -07003221 delalloc_start = delalloc_end + 1;
3222 continue;
3223 }
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003224 ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
3225 delalloc_end, &page_started, nr_written, wbc);
Chris Mason40f76582014-05-21 13:35:51 -07003226 /* File system has been set read-only */
3227 if (ret) {
3228 SetPageError(page);
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003229 /*
3230 * btrfs_run_delalloc_range should return < 0 for error
3231 * but just in case, we use > 0 here meaning the IO is
3232 * started, so we don't want to return > 0 unless
3233 * things are going well.
Chris Mason40f76582014-05-21 13:35:51 -07003234 */
3235 ret = ret < 0 ? ret : -EIO;
3236 goto done;
3237 }
3238 /*
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003239 * delalloc_end is already one less than the total length, so
3240 * we don't subtract one from PAGE_SIZE
Chris Mason40f76582014-05-21 13:35:51 -07003241 */
3242 delalloc_to_write += (delalloc_end - delalloc_start +
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003243 PAGE_SIZE) >> PAGE_SHIFT;
Chris Mason40f76582014-05-21 13:35:51 -07003244 delalloc_start = delalloc_end + 1;
3245 }
3246 if (wbc->nr_to_write < delalloc_to_write) {
3247 int thresh = 8192;
3248
3249 if (delalloc_to_write < thresh * 2)
3250 thresh = delalloc_to_write;
3251 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3252 thresh);
3253 }
3254
3255 /* did the fill delalloc function already unlock and start
3256 * the IO?
3257 */
3258 if (page_started) {
3259 /*
3260 * we've unlocked the page, so we can't update
3261 * the mapping's writeback index, just update
3262 * nr_to_write.
3263 */
3264 wbc->nr_to_write -= *nr_written;
3265 return 1;
3266 }
3267
3268 ret = 0;
3269
3270done:
3271 return ret;
3272}
3273
3274/*
3275 * helper for __extent_writepage. This calls the writepage start hooks,
3276 * and does the loop to map the page into extents and bios.
3277 *
3278 * We return 1 if the IO is started and the page is unlocked,
3279 * 0 if all went well (page still locked)
3280 * < 0 if there were errors (page still locked)
3281 */
3282static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3283 struct page *page,
3284 struct writeback_control *wbc,
3285 struct extent_page_data *epd,
3286 loff_t i_size,
3287 unsigned long nr_written,
David Sterbaf1c77c52017-06-06 19:03:49 +02003288 unsigned int write_flags, int *nr_ret)
Chris Mason40f76582014-05-21 13:35:51 -07003289{
Chris Masond1310b22008-01-24 16:13:08 -05003290 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003291 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003292 u64 page_end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05003293 u64 end;
3294 u64 cur = start;
3295 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003296 u64 block_start;
3297 u64 iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003298 struct extent_map *em;
3299 struct block_device *bdev;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003300 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003301 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003302 int ret = 0;
3303 int nr = 0;
3304 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003305
Nikolay Borisovd75855b2018-11-01 14:09:47 +02003306 ret = btrfs_writepage_cow_fixup(page, start, page_end);
3307 if (ret) {
3308 /* Fixup worker will requeue */
3309 if (ret == -EBUSY)
3310 wbc->pages_skipped++;
3311 else
3312 redirty_page_for_writepage(wbc, page);
Chris Mason40f76582014-05-21 13:35:51 -07003313
Nikolay Borisovd75855b2018-11-01 14:09:47 +02003314 update_nr_written(wbc, nr_written);
3315 unlock_page(page);
3316 return 1;
Chris Mason247e7432008-07-17 12:53:51 -04003317 }
3318
Chris Mason11c83492009-04-20 15:50:09 -04003319 /*
3320 * we don't want to touch the inode after unlocking the page,
3321 * so we update the mapping writeback index now
3322 */
David Sterba3d4b9492017-02-10 19:33:41 +01003323 update_nr_written(wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003324
Chris Masond1310b22008-01-24 16:13:08 -05003325 end = page_end;
Chris Mason40f76582014-05-21 13:35:51 -07003326 if (i_size <= start) {
Nikolay Borisovc6297322018-11-08 10:18:08 +02003327 btrfs_writepage_endio_finish_ordered(page, start, page_end, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003328 goto done;
3329 }
3330
Chris Masond1310b22008-01-24 16:13:08 -05003331 blocksize = inode->i_sb->s_blocksize;
3332
3333 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003334 u64 em_end;
David Sterba6273b7f2017-10-04 17:30:11 +02003335 u64 offset;
David Sterba58409ed2016-05-04 11:46:10 +02003336
Chris Mason40f76582014-05-21 13:35:51 -07003337 if (cur >= i_size) {
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02003338 btrfs_writepage_endio_finish_ordered(page, cur,
Nikolay Borisovc6297322018-11-08 10:18:08 +02003339 page_end, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003340 break;
3341 }
David Sterba3c98c622017-06-23 04:01:08 +02003342 em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003343 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003344 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003345 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003346 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003347 break;
3348 }
3349
3350 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003351 em_end = extent_map_end(em);
3352 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003353 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003354 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003355 iosize = ALIGN(iosize, blocksize);
David Sterba6273b7f2017-10-04 17:30:11 +02003356 offset = em->block_start + extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003357 bdev = em->bdev;
3358 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003359 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003360 free_extent_map(em);
3361 em = NULL;
3362
Chris Masonc8b97812008-10-29 14:49:59 -04003363 /*
3364 * compressed and inline extents are written through other
3365 * paths in the FS
3366 */
3367 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003368 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003369 /*
3370 * end_io notification does not happen here for
3371 * compressed extents
3372 */
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02003373 if (!compressed)
3374 btrfs_writepage_endio_finish_ordered(page, cur,
3375 cur + iosize - 1,
Nikolay Borisovc6297322018-11-08 10:18:08 +02003376 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003377 else if (compressed) {
3378 /* we don't want to end_page_writeback on
3379 * a compressed extent. this happens
3380 * elsewhere
3381 */
3382 nr++;
3383 }
3384
3385 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003386 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003387 continue;
3388 }
Chris Masonc8b97812008-10-29 14:49:59 -04003389
David Sterba5cdc84b2018-07-18 20:32:52 +02003390 btrfs_set_range_writeback(tree, cur, cur + iosize - 1);
David Sterba58409ed2016-05-04 11:46:10 +02003391 if (!PageWriteback(page)) {
3392 btrfs_err(BTRFS_I(inode)->root->fs_info,
3393 "page %lu not writeback, cur %llu end %llu",
3394 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003395 }
David Sterba58409ed2016-05-04 11:46:10 +02003396
David Sterba4b81ba42017-06-06 19:14:26 +02003397 ret = submit_extent_page(REQ_OP_WRITE | write_flags, tree, wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02003398 page, offset, iosize, pg_offset,
David Sterbac2df8bb2017-02-10 19:29:38 +01003399 bdev, &epd->bio,
David Sterba58409ed2016-05-04 11:46:10 +02003400 end_bio_extent_writepage,
3401 0, 0, 0, false);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003402 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003403 SetPageError(page);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003404 if (PageWriteback(page))
3405 end_page_writeback(page);
3406 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003407
Chris Masond1310b22008-01-24 16:13:08 -05003408 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003409 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003410 nr++;
3411 }
3412done:
Chris Mason40f76582014-05-21 13:35:51 -07003413 *nr_ret = nr;
Chris Mason40f76582014-05-21 13:35:51 -07003414 return ret;
3415}
3416
3417/*
3418 * the writepage semantics are similar to regular writepage. extent
3419 * records are inserted to lock ranges in the tree, and as dirty areas
3420 * are found, they are marked writeback. Then the lock bits are removed
3421 * and the end_io handler clears the writeback ranges
3422 */
3423static int __extent_writepage(struct page *page, struct writeback_control *wbc,
David Sterbaaab6e9e2017-11-30 18:00:02 +01003424 struct extent_page_data *epd)
Chris Mason40f76582014-05-21 13:35:51 -07003425{
3426 struct inode *inode = page->mapping->host;
Chris Mason40f76582014-05-21 13:35:51 -07003427 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003428 u64 page_end = start + PAGE_SIZE - 1;
Chris Mason40f76582014-05-21 13:35:51 -07003429 int ret;
3430 int nr = 0;
3431 size_t pg_offset = 0;
3432 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003433 unsigned long end_index = i_size >> PAGE_SHIFT;
David Sterbaf1c77c52017-06-06 19:03:49 +02003434 unsigned int write_flags = 0;
Chris Mason40f76582014-05-21 13:35:51 -07003435 unsigned long nr_written = 0;
3436
Liu Boff40adf2017-08-24 18:19:48 -06003437 write_flags = wbc_to_write_flags(wbc);
Chris Mason40f76582014-05-21 13:35:51 -07003438
3439 trace___extent_writepage(page, inode, wbc);
3440
3441 WARN_ON(!PageLocked(page));
3442
3443 ClearPageError(page);
3444
Johannes Thumshirn70730172018-12-05 15:23:03 +01003445 pg_offset = offset_in_page(i_size);
Chris Mason40f76582014-05-21 13:35:51 -07003446 if (page->index > end_index ||
3447 (page->index == end_index && !pg_offset)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003448 page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
Chris Mason40f76582014-05-21 13:35:51 -07003449 unlock_page(page);
3450 return 0;
3451 }
3452
3453 if (page->index == end_index) {
3454 char *userpage;
3455
3456 userpage = kmap_atomic(page);
3457 memset(userpage + pg_offset, 0,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003458 PAGE_SIZE - pg_offset);
Chris Mason40f76582014-05-21 13:35:51 -07003459 kunmap_atomic(userpage);
3460 flush_dcache_page(page);
3461 }
3462
3463 pg_offset = 0;
3464
3465 set_page_extent_mapped(page);
3466
Nikolay Borisov7789a552018-11-08 10:18:06 +02003467 if (!epd->extent_locked) {
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003468 ret = writepage_delalloc(inode, page, wbc, start, &nr_written);
Nikolay Borisov7789a552018-11-08 10:18:06 +02003469 if (ret == 1)
3470 goto done_unlocked;
3471 if (ret)
3472 goto done;
3473 }
Chris Mason40f76582014-05-21 13:35:51 -07003474
3475 ret = __extent_writepage_io(inode, page, wbc, epd,
3476 i_size, nr_written, write_flags, &nr);
3477 if (ret == 1)
3478 goto done_unlocked;
3479
3480done:
Chris Masond1310b22008-01-24 16:13:08 -05003481 if (nr == 0) {
3482 /* make sure the mapping tag for page dirty gets cleared */
3483 set_page_writeback(page);
3484 end_page_writeback(page);
3485 }
Filipe Manana61391d52014-05-09 17:17:40 +01003486 if (PageError(page)) {
3487 ret = ret < 0 ? ret : -EIO;
3488 end_extent_writepage(page, ret, start, page_end);
3489 }
Chris Masond1310b22008-01-24 16:13:08 -05003490 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003491 return ret;
Chris Mason771ed682008-11-06 22:02:51 -05003492
Chris Mason11c83492009-04-20 15:50:09 -04003493done_unlocked:
Chris Masond1310b22008-01-24 16:13:08 -05003494 return 0;
3495}
3496
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003497void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003498{
NeilBrown74316202014-07-07 15:16:04 +10003499 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3500 TASK_UNINTERRUPTIBLE);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003501}
3502
Chris Mason0e378df2014-05-19 20:55:27 -07003503static noinline_for_stack int
3504lock_extent_buffer_for_io(struct extent_buffer *eb,
3505 struct btrfs_fs_info *fs_info,
3506 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003507{
David Sterbacc5e31a2018-03-01 18:20:27 +01003508 int i, num_pages;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003509 int flush = 0;
3510 int ret = 0;
3511
3512 if (!btrfs_try_tree_write_lock(eb)) {
3513 flush = 1;
3514 flush_write_bio(epd);
3515 btrfs_tree_lock(eb);
3516 }
3517
3518 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3519 btrfs_tree_unlock(eb);
3520 if (!epd->sync_io)
3521 return 0;
3522 if (!flush) {
3523 flush_write_bio(epd);
3524 flush = 1;
3525 }
Chris Masona098d8e82012-03-21 12:09:56 -04003526 while (1) {
3527 wait_on_extent_buffer_writeback(eb);
3528 btrfs_tree_lock(eb);
3529 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3530 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003531 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003532 }
3533 }
3534
Josef Bacik51561ff2012-07-20 16:25:24 -04003535 /*
3536 * We need to do this to prevent races in people who check if the eb is
3537 * under IO since we can end up having no IO bits set for a short period
3538 * of time.
3539 */
3540 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003541 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3542 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003543 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003544 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Nikolay Borisov104b4e52017-06-20 21:01:20 +03003545 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
3546 -eb->len,
3547 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003548 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003549 } else {
3550 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003551 }
3552
3553 btrfs_tree_unlock(eb);
3554
3555 if (!ret)
3556 return ret;
3557
David Sterba65ad0102018-06-29 10:56:49 +02003558 num_pages = num_extent_pages(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003559 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003560 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003561
3562 if (!trylock_page(p)) {
3563 if (!flush) {
3564 flush_write_bio(epd);
3565 flush = 1;
3566 }
3567 lock_page(p);
3568 }
3569 }
3570
3571 return ret;
3572}
3573
3574static void end_extent_buffer_writeback(struct extent_buffer *eb)
3575{
3576 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003577 smp_mb__after_atomic();
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003578 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3579}
3580
Filipe Manana656f30d2014-09-26 12:25:56 +01003581static void set_btree_ioerr(struct page *page)
3582{
3583 struct extent_buffer *eb = (struct extent_buffer *)page->private;
Filipe Manana656f30d2014-09-26 12:25:56 +01003584
3585 SetPageError(page);
3586 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3587 return;
3588
3589 /*
3590 * If writeback for a btree extent that doesn't belong to a log tree
3591 * failed, increment the counter transaction->eb_write_errors.
3592 * We do this because while the transaction is running and before it's
3593 * committing (when we call filemap_fdata[write|wait]_range against
3594 * the btree inode), we might have
3595 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3596 * returns an error or an error happens during writeback, when we're
3597 * committing the transaction we wouldn't know about it, since the pages
3598 * can be no longer dirty nor marked anymore for writeback (if a
3599 * subsequent modification to the extent buffer didn't happen before the
3600 * transaction commit), which makes filemap_fdata[write|wait]_range not
3601 * able to find the pages tagged with SetPageError at transaction
3602 * commit time. So if this happens we must abort the transaction,
3603 * otherwise we commit a super block with btree roots that point to
3604 * btree nodes/leafs whose content on disk is invalid - either garbage
3605 * or the content of some node/leaf from a past generation that got
3606 * cowed or deleted and is no longer valid.
3607 *
3608 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3609 * not be enough - we need to distinguish between log tree extents vs
3610 * non-log tree extents, and the next filemap_fdatawait_range() call
3611 * will catch and clear such errors in the mapping - and that call might
3612 * be from a log sync and not from a transaction commit. Also, checking
3613 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3614 * not done and would not be reliable - the eb might have been released
3615 * from memory and reading it back again means that flag would not be
3616 * set (since it's a runtime flag, not persisted on disk).
3617 *
3618 * Using the flags below in the btree inode also makes us achieve the
3619 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3620 * writeback for all dirty pages and before filemap_fdatawait_range()
3621 * is called, the writeback for all dirty pages had already finished
3622 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3623 * filemap_fdatawait_range() would return success, as it could not know
3624 * that writeback errors happened (the pages were no longer tagged for
3625 * writeback).
3626 */
3627 switch (eb->log_index) {
3628 case -1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003629 set_bit(BTRFS_FS_BTREE_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003630 break;
3631 case 0:
Josef Bacikafcdd122016-09-02 15:40:02 -04003632 set_bit(BTRFS_FS_LOG1_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003633 break;
3634 case 1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003635 set_bit(BTRFS_FS_LOG2_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003636 break;
3637 default:
3638 BUG(); /* unexpected, logic error */
3639 }
3640}
3641
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003642static void end_bio_extent_buffer_writepage(struct bio *bio)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003643{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003644 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003645 struct extent_buffer *eb;
Kent Overstreet2c30c712013-11-07 12:20:26 -08003646 int i, done;
Ming Lei6dc4f102019-02-15 19:13:19 +08003647 struct bvec_iter_all iter_all;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003648
David Sterbac09abff2017-07-13 18:10:07 +02003649 ASSERT(!bio_flagged(bio, BIO_CLONED));
Ming Lei6dc4f102019-02-15 19:13:19 +08003650 bio_for_each_segment_all(bvec, bio, i, iter_all) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003651 struct page *page = bvec->bv_page;
3652
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003653 eb = (struct extent_buffer *)page->private;
3654 BUG_ON(!eb);
3655 done = atomic_dec_and_test(&eb->io_pages);
3656
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02003657 if (bio->bi_status ||
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003658 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003659 ClearPageUptodate(page);
Filipe Manana656f30d2014-09-26 12:25:56 +01003660 set_btree_ioerr(page);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003661 }
3662
3663 end_page_writeback(page);
3664
3665 if (!done)
3666 continue;
3667
3668 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003669 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003670
3671 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003672}
3673
Chris Mason0e378df2014-05-19 20:55:27 -07003674static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003675 struct btrfs_fs_info *fs_info,
3676 struct writeback_control *wbc,
3677 struct extent_page_data *epd)
3678{
3679 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
Josef Bacikf28491e2013-12-16 13:24:27 -05003680 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003681 u64 offset = eb->start;
Liu Bo851cd172016-09-23 13:44:44 -07003682 u32 nritems;
David Sterbacc5e31a2018-03-01 18:20:27 +01003683 int i, num_pages;
Liu Bo851cd172016-09-23 13:44:44 -07003684 unsigned long start, end;
Liu Boff40adf2017-08-24 18:19:48 -06003685 unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003686 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003687
Filipe Manana656f30d2014-09-26 12:25:56 +01003688 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02003689 num_pages = num_extent_pages(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003690 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003691
Liu Bo851cd172016-09-23 13:44:44 -07003692 /* set btree blocks beyond nritems with 0 to avoid stale content. */
3693 nritems = btrfs_header_nritems(eb);
Liu Bo3eb548e2016-09-14 17:22:57 -07003694 if (btrfs_header_level(eb) > 0) {
Liu Bo3eb548e2016-09-14 17:22:57 -07003695 end = btrfs_node_key_ptr_offset(nritems);
3696
David Sterbab159fa22016-11-08 18:09:03 +01003697 memzero_extent_buffer(eb, end, eb->len - end);
Liu Bo851cd172016-09-23 13:44:44 -07003698 } else {
3699 /*
3700 * leaf:
3701 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
3702 */
3703 start = btrfs_item_nr_offset(nritems);
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003704 end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(fs_info, eb);
David Sterbab159fa22016-11-08 18:09:03 +01003705 memzero_extent_buffer(eb, start, end - start);
Liu Bo3eb548e2016-09-14 17:22:57 -07003706 }
3707
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003708 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003709 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003710
3711 clear_page_dirty_for_io(p);
3712 set_page_writeback(p);
David Sterba4b81ba42017-06-06 19:14:26 +02003713 ret = submit_extent_page(REQ_OP_WRITE | write_flags, tree, wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02003714 p, offset, PAGE_SIZE, 0, bdev,
David Sterbac2df8bb2017-02-10 19:29:38 +01003715 &epd->bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003716 end_bio_extent_buffer_writepage,
Liu Bo18fdc672017-09-13 12:18:22 -06003717 0, 0, 0, false);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003718 if (ret) {
Filipe Manana656f30d2014-09-26 12:25:56 +01003719 set_btree_ioerr(p);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003720 if (PageWriteback(p))
3721 end_page_writeback(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003722 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3723 end_extent_buffer_writeback(eb);
3724 ret = -EIO;
3725 break;
3726 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003727 offset += PAGE_SIZE;
David Sterba3d4b9492017-02-10 19:33:41 +01003728 update_nr_written(wbc, 1);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003729 unlock_page(p);
3730 }
3731
3732 if (unlikely(ret)) {
3733 for (; i < num_pages; i++) {
Chris Masonbbf65cf2014-10-04 09:56:45 -07003734 struct page *p = eb->pages[i];
Liu Bo81465022014-09-23 22:22:33 +08003735 clear_page_dirty_for_io(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003736 unlock_page(p);
3737 }
3738 }
3739
3740 return ret;
3741}
3742
3743int btree_write_cache_pages(struct address_space *mapping,
3744 struct writeback_control *wbc)
3745{
3746 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3747 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3748 struct extent_buffer *eb, *prev_eb = NULL;
3749 struct extent_page_data epd = {
3750 .bio = NULL,
3751 .tree = tree,
3752 .extent_locked = 0,
3753 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3754 };
3755 int ret = 0;
3756 int done = 0;
3757 int nr_to_write_done = 0;
3758 struct pagevec pvec;
3759 int nr_pages;
3760 pgoff_t index;
3761 pgoff_t end; /* Inclusive */
3762 int scanned = 0;
Matthew Wilcox10bbd232017-12-05 17:30:38 -05003763 xa_mark_t tag;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003764
Mel Gorman86679822017-11-15 17:37:52 -08003765 pagevec_init(&pvec);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003766 if (wbc->range_cyclic) {
3767 index = mapping->writeback_index; /* Start from prev offset */
3768 end = -1;
3769 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003770 index = wbc->range_start >> PAGE_SHIFT;
3771 end = wbc->range_end >> PAGE_SHIFT;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003772 scanned = 1;
3773 }
3774 if (wbc->sync_mode == WB_SYNC_ALL)
3775 tag = PAGECACHE_TAG_TOWRITE;
3776 else
3777 tag = PAGECACHE_TAG_DIRTY;
3778retry:
3779 if (wbc->sync_mode == WB_SYNC_ALL)
3780 tag_pages_for_writeback(mapping, index, end);
3781 while (!done && !nr_to_write_done && (index <= end) &&
Jan Kara4006f432017-11-15 17:34:37 -08003782 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
Jan Kara67fd7072017-11-15 17:35:19 -08003783 tag))) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003784 unsigned i;
3785
3786 scanned = 1;
3787 for (i = 0; i < nr_pages; i++) {
3788 struct page *page = pvec.pages[i];
3789
3790 if (!PagePrivate(page))
3791 continue;
3792
Josef Bacikb5bae262012-09-14 13:43:01 -04003793 spin_lock(&mapping->private_lock);
3794 if (!PagePrivate(page)) {
3795 spin_unlock(&mapping->private_lock);
3796 continue;
3797 }
3798
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003799 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003800
3801 /*
3802 * Shouldn't happen and normally this would be a BUG_ON
3803 * but no sense in crashing the users box for something
3804 * we can survive anyway.
3805 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303806 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003807 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003808 continue;
3809 }
3810
Josef Bacikb5bae262012-09-14 13:43:01 -04003811 if (eb == prev_eb) {
3812 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003813 continue;
3814 }
3815
Josef Bacikb5bae262012-09-14 13:43:01 -04003816 ret = atomic_inc_not_zero(&eb->refs);
3817 spin_unlock(&mapping->private_lock);
3818 if (!ret)
3819 continue;
3820
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003821 prev_eb = eb;
3822 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3823 if (!ret) {
3824 free_extent_buffer(eb);
3825 continue;
3826 }
3827
3828 ret = write_one_eb(eb, fs_info, wbc, &epd);
3829 if (ret) {
3830 done = 1;
3831 free_extent_buffer(eb);
3832 break;
3833 }
3834 free_extent_buffer(eb);
3835
3836 /*
3837 * the filesystem may choose to bump up nr_to_write.
3838 * We have to make sure to honor the new nr_to_write
3839 * at any time
3840 */
3841 nr_to_write_done = wbc->nr_to_write <= 0;
3842 }
3843 pagevec_release(&pvec);
3844 cond_resched();
3845 }
3846 if (!scanned && !done) {
3847 /*
3848 * We hit the last page and there is more work to be done: wrap
3849 * back to the start of the file
3850 */
3851 scanned = 1;
3852 index = 0;
3853 goto retry;
3854 }
3855 flush_write_bio(&epd);
3856 return ret;
3857}
3858
Chris Masond1310b22008-01-24 16:13:08 -05003859/**
Chris Mason4bef0842008-09-08 11:18:08 -04003860 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
Chris Masond1310b22008-01-24 16:13:08 -05003861 * @mapping: address space structure to write
3862 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
David Sterba935db852017-06-23 04:30:28 +02003863 * @data: data passed to __extent_writepage function
Chris Masond1310b22008-01-24 16:13:08 -05003864 *
3865 * If a page is already under I/O, write_cache_pages() skips it, even
3866 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3867 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3868 * and msync() need to guarantee that all the data which was dirty at the time
3869 * the call was made get new I/O started against them. If wbc->sync_mode is
3870 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3871 * existing IO to complete.
3872 */
David Sterba4242b642017-02-10 19:38:24 +01003873static int extent_write_cache_pages(struct address_space *mapping,
Chris Mason4bef0842008-09-08 11:18:08 -04003874 struct writeback_control *wbc,
David Sterbaaab6e9e2017-11-30 18:00:02 +01003875 struct extent_page_data *epd)
Chris Masond1310b22008-01-24 16:13:08 -05003876{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003877 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003878 int ret = 0;
3879 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003880 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003881 struct pagevec pvec;
3882 int nr_pages;
3883 pgoff_t index;
3884 pgoff_t end; /* Inclusive */
Liu Boa91326672016-03-07 16:56:21 -08003885 pgoff_t done_index;
3886 int range_whole = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003887 int scanned = 0;
Matthew Wilcox10bbd232017-12-05 17:30:38 -05003888 xa_mark_t tag;
Chris Masond1310b22008-01-24 16:13:08 -05003889
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003890 /*
3891 * We have to hold onto the inode so that ordered extents can do their
3892 * work when the IO finishes. The alternative to this is failing to add
3893 * an ordered extent if the igrab() fails there and that is a huge pain
3894 * to deal with, so instead just hold onto the inode throughout the
3895 * writepages operation. If it fails here we are freeing up the inode
3896 * anyway and we'd rather not waste our time writing out stuff that is
3897 * going to be truncated anyway.
3898 */
3899 if (!igrab(inode))
3900 return 0;
3901
Mel Gorman86679822017-11-15 17:37:52 -08003902 pagevec_init(&pvec);
Chris Masond1310b22008-01-24 16:13:08 -05003903 if (wbc->range_cyclic) {
3904 index = mapping->writeback_index; /* Start from prev offset */
3905 end = -1;
3906 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003907 index = wbc->range_start >> PAGE_SHIFT;
3908 end = wbc->range_end >> PAGE_SHIFT;
Liu Boa91326672016-03-07 16:56:21 -08003909 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
3910 range_whole = 1;
Chris Masond1310b22008-01-24 16:13:08 -05003911 scanned = 1;
3912 }
Ethan Lien3cd24c62018-11-01 14:49:03 +08003913
3914 /*
3915 * We do the tagged writepage as long as the snapshot flush bit is set
3916 * and we are the first one who do the filemap_flush() on this inode.
3917 *
3918 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
3919 * not race in and drop the bit.
3920 */
3921 if (range_whole && wbc->nr_to_write == LONG_MAX &&
3922 test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
3923 &BTRFS_I(inode)->runtime_flags))
3924 wbc->tagged_writepages = 1;
3925
3926 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Josef Bacikf7aaa062011-07-15 21:26:38 +00003927 tag = PAGECACHE_TAG_TOWRITE;
3928 else
3929 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003930retry:
Ethan Lien3cd24c62018-11-01 14:49:03 +08003931 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Josef Bacikf7aaa062011-07-15 21:26:38 +00003932 tag_pages_for_writeback(mapping, index, end);
Liu Boa91326672016-03-07 16:56:21 -08003933 done_index = index;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003934 while (!done && !nr_to_write_done && (index <= end) &&
Jan Kara67fd7072017-11-15 17:35:19 -08003935 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping,
3936 &index, end, tag))) {
Chris Masond1310b22008-01-24 16:13:08 -05003937 unsigned i;
3938
3939 scanned = 1;
3940 for (i = 0; i < nr_pages; i++) {
3941 struct page *page = pvec.pages[i];
3942
Liu Boa91326672016-03-07 16:56:21 -08003943 done_index = page->index;
Chris Masond1310b22008-01-24 16:13:08 -05003944 /*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07003945 * At this point we hold neither the i_pages lock nor
3946 * the page lock: the page may be truncated or
3947 * invalidated (changing page->mapping to NULL),
3948 * or even swizzled back from swapper_space to
3949 * tmpfs file mapping
Chris Masond1310b22008-01-24 16:13:08 -05003950 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05003951 if (!trylock_page(page)) {
David Sterbaaab6e9e2017-11-30 18:00:02 +01003952 flush_write_bio(epd);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003953 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04003954 }
Chris Masond1310b22008-01-24 16:13:08 -05003955
3956 if (unlikely(page->mapping != mapping)) {
3957 unlock_page(page);
3958 continue;
3959 }
3960
Chris Masond2c3f4f2008-11-19 12:44:22 -05003961 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05003962 if (PageWriteback(page))
David Sterbaaab6e9e2017-11-30 18:00:02 +01003963 flush_write_bio(epd);
Chris Masond1310b22008-01-24 16:13:08 -05003964 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003965 }
Chris Masond1310b22008-01-24 16:13:08 -05003966
3967 if (PageWriteback(page) ||
3968 !clear_page_dirty_for_io(page)) {
3969 unlock_page(page);
3970 continue;
3971 }
3972
David Sterbaaab6e9e2017-11-30 18:00:02 +01003973 ret = __extent_writepage(page, wbc, epd);
Chris Masond1310b22008-01-24 16:13:08 -05003974
3975 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3976 unlock_page(page);
3977 ret = 0;
3978 }
Liu Boa91326672016-03-07 16:56:21 -08003979 if (ret < 0) {
3980 /*
3981 * done_index is set past this page,
3982 * so media errors will not choke
3983 * background writeout for the entire
3984 * file. This has consequences for
3985 * range_cyclic semantics (ie. it may
3986 * not be suitable for data integrity
3987 * writeout).
3988 */
3989 done_index = page->index + 1;
3990 done = 1;
3991 break;
3992 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003993
3994 /*
3995 * the filesystem may choose to bump up nr_to_write.
3996 * We have to make sure to honor the new nr_to_write
3997 * at any time
3998 */
3999 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05004000 }
4001 pagevec_release(&pvec);
4002 cond_resched();
4003 }
Liu Bo894b36e2016-03-07 16:56:22 -08004004 if (!scanned && !done) {
Chris Masond1310b22008-01-24 16:13:08 -05004005 /*
4006 * We hit the last page and there is more work to be done: wrap
4007 * back to the start of the file
4008 */
4009 scanned = 1;
4010 index = 0;
4011 goto retry;
4012 }
Liu Boa91326672016-03-07 16:56:21 -08004013
4014 if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
4015 mapping->writeback_index = done_index;
4016
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004017 btrfs_add_delayed_iput(inode);
Liu Bo894b36e2016-03-07 16:56:22 -08004018 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05004019}
Chris Masond1310b22008-01-24 16:13:08 -05004020
Nikolay Borisov0a9b0e52017-12-08 15:55:59 +02004021int extent_write_full_page(struct page *page, struct writeback_control *wbc)
Chris Masond1310b22008-01-24 16:13:08 -05004022{
4023 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004024 struct extent_page_data epd = {
4025 .bio = NULL,
Nikolay Borisov0a9b0e52017-12-08 15:55:59 +02004026 .tree = &BTRFS_I(page->mapping->host)->io_tree,
Chris Mason771ed682008-11-06 22:02:51 -05004027 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004028 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05004029 };
Chris Masond1310b22008-01-24 16:13:08 -05004030
Chris Masond1310b22008-01-24 16:13:08 -05004031 ret = __extent_writepage(page, wbc, &epd);
4032
David Sterbae2932ee2017-06-23 04:16:17 +02004033 flush_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004034 return ret;
4035}
Chris Masond1310b22008-01-24 16:13:08 -05004036
Nikolay Borisov5e3ee232017-12-08 15:55:58 +02004037int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
Chris Mason771ed682008-11-06 22:02:51 -05004038 int mode)
4039{
4040 int ret = 0;
4041 struct address_space *mapping = inode->i_mapping;
Nikolay Borisov5e3ee232017-12-08 15:55:58 +02004042 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Chris Mason771ed682008-11-06 22:02:51 -05004043 struct page *page;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004044 unsigned long nr_pages = (end - start + PAGE_SIZE) >>
4045 PAGE_SHIFT;
Chris Mason771ed682008-11-06 22:02:51 -05004046
4047 struct extent_page_data epd = {
4048 .bio = NULL,
4049 .tree = tree,
Chris Mason771ed682008-11-06 22:02:51 -05004050 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04004051 .sync_io = mode == WB_SYNC_ALL,
Chris Mason771ed682008-11-06 22:02:51 -05004052 };
4053 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05004054 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05004055 .nr_to_write = nr_pages * 2,
4056 .range_start = start,
4057 .range_end = end + 1,
4058 };
4059
Chris Masond3977122009-01-05 21:25:51 -05004060 while (start <= end) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004061 page = find_get_page(mapping, start >> PAGE_SHIFT);
Chris Mason771ed682008-11-06 22:02:51 -05004062 if (clear_page_dirty_for_io(page))
4063 ret = __extent_writepage(page, &wbc_writepages, &epd);
4064 else {
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02004065 btrfs_writepage_endio_finish_ordered(page, start,
Nikolay Borisovc6297322018-11-08 10:18:08 +02004066 start + PAGE_SIZE - 1, 1);
Chris Mason771ed682008-11-06 22:02:51 -05004067 unlock_page(page);
4068 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004069 put_page(page);
4070 start += PAGE_SIZE;
Chris Mason771ed682008-11-06 22:02:51 -05004071 }
4072
David Sterbae2932ee2017-06-23 04:16:17 +02004073 flush_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05004074 return ret;
4075}
Chris Masond1310b22008-01-24 16:13:08 -05004076
Nikolay Borisov8ae225a2018-04-19 10:46:38 +03004077int extent_writepages(struct address_space *mapping,
Chris Masond1310b22008-01-24 16:13:08 -05004078 struct writeback_control *wbc)
4079{
4080 int ret = 0;
4081 struct extent_page_data epd = {
4082 .bio = NULL,
Nikolay Borisov8ae225a2018-04-19 10:46:38 +03004083 .tree = &BTRFS_I(mapping->host)->io_tree,
Chris Mason771ed682008-11-06 22:02:51 -05004084 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004085 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05004086 };
4087
David Sterba935db852017-06-23 04:30:28 +02004088 ret = extent_write_cache_pages(mapping, wbc, &epd);
David Sterbae2932ee2017-06-23 04:16:17 +02004089 flush_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004090 return ret;
4091}
Chris Masond1310b22008-01-24 16:13:08 -05004092
Nikolay Borisov2a3ff0a2018-04-19 10:46:36 +03004093int extent_readpages(struct address_space *mapping, struct list_head *pages,
4094 unsigned nr_pages)
Chris Masond1310b22008-01-24 16:13:08 -05004095{
4096 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04004097 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06004098 struct page *pagepool[16];
Miao Xie125bac012013-07-25 19:22:37 +08004099 struct extent_map *em_cached = NULL;
Nikolay Borisov2a3ff0a2018-04-19 10:46:36 +03004100 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
Liu Bo67c96842012-07-20 21:43:09 -06004101 int nr = 0;
Filipe Manana808f80b2015-09-28 09:56:26 +01004102 u64 prev_em_start = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05004103
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004104 while (!list_empty(pages)) {
4105 for (nr = 0; nr < ARRAY_SIZE(pagepool) && !list_empty(pages);) {
Nikolay Borisovf86196e2019-01-03 15:29:02 -08004106 struct page *page = lru_to_page(pages);
Chris Masond1310b22008-01-24 16:13:08 -05004107
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004108 prefetchw(&page->flags);
4109 list_del(&page->lru);
4110 if (add_to_page_cache_lru(page, mapping, page->index,
4111 readahead_gfp_mask(mapping))) {
4112 put_page(page);
4113 continue;
4114 }
4115
4116 pagepool[nr++] = page;
Chris Masond1310b22008-01-24 16:13:08 -05004117 }
Liu Bo67c96842012-07-20 21:43:09 -06004118
David Sterbae4d17ef2017-06-23 04:09:57 +02004119 __extent_readpages(tree, pagepool, nr, &em_cached, &bio,
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004120 &bio_flags, &prev_em_start);
Chris Masond1310b22008-01-24 16:13:08 -05004121 }
Liu Bo67c96842012-07-20 21:43:09 -06004122
Miao Xie125bac012013-07-25 19:22:37 +08004123 if (em_cached)
4124 free_extent_map(em_cached);
4125
Chris Masond1310b22008-01-24 16:13:08 -05004126 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05004127 return submit_one_bio(bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004128 return 0;
4129}
Chris Masond1310b22008-01-24 16:13:08 -05004130
4131/*
4132 * basic invalidatepage code, this waits on any locked or writeback
4133 * ranges corresponding to the page, and then deletes any extent state
4134 * records from the tree
4135 */
4136int extent_invalidatepage(struct extent_io_tree *tree,
4137 struct page *page, unsigned long offset)
4138{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004139 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004140 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004141 u64 end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05004142 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4143
Qu Wenruofda28322013-02-26 08:10:22 +00004144 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004145 if (start > end)
4146 return 0;
4147
David Sterbaff13db42015-12-03 14:30:40 +01004148 lock_extent_bits(tree, start, end, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004149 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05004150 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04004151 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4152 EXTENT_DO_ACCOUNTING,
David Sterbaae0f1622017-10-31 16:37:52 +01004153 1, 1, &cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05004154 return 0;
4155}
Chris Masond1310b22008-01-24 16:13:08 -05004156
4157/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004158 * a helper for releasepage, this tests for areas of the page that
4159 * are locked or under IO and drops the related state bits if it is safe
4160 * to drop the page.
4161 */
Nikolay Borisov29c68b2d2018-04-19 10:46:35 +03004162static int try_release_extent_state(struct extent_io_tree *tree,
Eric Sandeen48a3b632013-04-25 20:41:01 +00004163 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004164{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004165 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004166 u64 end = start + PAGE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004167 int ret = 1;
4168
Chris Mason211f90e2008-07-18 11:56:15 -04004169 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04004170 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04004171 ret = 0;
4172 else {
Chris Mason11ef1602009-09-23 20:28:46 -04004173 /*
4174 * at this point we can safely clear everything except the
4175 * locked bit and the nodatasum bit
4176 */
David Sterba66b0c882017-10-31 16:30:47 +01004177 ret = __clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004178 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
David Sterba66b0c882017-10-31 16:30:47 +01004179 0, 0, NULL, mask, NULL);
Chris Masone3f24cc2011-02-14 12:52:08 -05004180
4181 /* if clear_extent_bit failed for enomem reasons,
4182 * we can't allow the release to continue.
4183 */
4184 if (ret < 0)
4185 ret = 0;
4186 else
4187 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004188 }
4189 return ret;
4190}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004191
4192/*
Chris Masond1310b22008-01-24 16:13:08 -05004193 * a helper for releasepage. As long as there are no locked extents
4194 * in the range corresponding to the page, both state records and extent
4195 * map records are removed
4196 */
Nikolay Borisov477a30b2018-04-19 10:46:34 +03004197int try_release_extent_mapping(struct page *page, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004198{
4199 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004200 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004201 u64 end = start + PAGE_SIZE - 1;
Filipe Mananabd3599a2018-07-12 01:36:43 +01004202 struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
4203 struct extent_io_tree *tree = &btrfs_inode->io_tree;
4204 struct extent_map_tree *map = &btrfs_inode->extent_tree;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004205
Mel Gormand0164ad2015-11-06 16:28:21 -08004206 if (gfpflags_allow_blocking(mask) &&
Byongho Leeee221842015-12-15 01:42:10 +09004207 page->mapping->host->i_size > SZ_16M) {
Yan39b56372008-02-15 10:40:50 -05004208 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004209 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004210 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004211 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004212 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004213 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004214 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004215 break;
4216 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004217 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4218 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004219 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004220 free_extent_map(em);
4221 break;
4222 }
4223 if (!test_range_bit(tree, em->start,
4224 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04004225 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04004226 0, NULL)) {
Filipe Mananabd3599a2018-07-12 01:36:43 +01004227 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4228 &btrfs_inode->runtime_flags);
Chris Mason70dec802008-01-29 09:59:12 -05004229 remove_extent_mapping(map, em);
4230 /* once for the rb tree */
4231 free_extent_map(em);
4232 }
4233 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004234 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004235
4236 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004237 free_extent_map(em);
4238 }
Chris Masond1310b22008-01-24 16:13:08 -05004239 }
Nikolay Borisov29c68b2d2018-04-19 10:46:35 +03004240 return try_release_extent_state(tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004241}
Chris Masond1310b22008-01-24 16:13:08 -05004242
Chris Masonec29ed52011-02-23 16:23:20 -05004243/*
4244 * helper function for fiemap, which doesn't want to see any holes.
4245 * This maps until we find something past 'last'
4246 */
4247static struct extent_map *get_extent_skip_holes(struct inode *inode,
David Sterbae3350e12017-06-23 04:09:57 +02004248 u64 offset, u64 last)
Chris Masonec29ed52011-02-23 16:23:20 -05004249{
Jeff Mahoneyda170662016-06-15 09:22:56 -04004250 u64 sectorsize = btrfs_inode_sectorsize(inode);
Chris Masonec29ed52011-02-23 16:23:20 -05004251 struct extent_map *em;
4252 u64 len;
4253
4254 if (offset >= last)
4255 return NULL;
4256
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304257 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004258 len = last - offset;
4259 if (len == 0)
4260 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004261 len = ALIGN(len, sectorsize);
Nikolay Borisov4ab47a82018-12-12 09:42:32 +02004262 em = btrfs_get_extent_fiemap(BTRFS_I(inode), offset, len);
David Sterbac7040052011-04-19 18:00:01 +02004263 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004264 return em;
4265
4266 /* if this isn't a hole return it */
Nikolay Borisov4a2d25c2017-11-23 10:51:43 +02004267 if (em->block_start != EXTENT_MAP_HOLE)
Chris Masonec29ed52011-02-23 16:23:20 -05004268 return em;
Chris Masonec29ed52011-02-23 16:23:20 -05004269
4270 /* this is a hole, advance to the next extent */
4271 offset = extent_map_end(em);
4272 free_extent_map(em);
4273 if (offset >= last)
4274 break;
4275 }
4276 return NULL;
4277}
4278
Qu Wenruo47518322017-04-07 10:43:15 +08004279/*
4280 * To cache previous fiemap extent
4281 *
4282 * Will be used for merging fiemap extent
4283 */
4284struct fiemap_cache {
4285 u64 offset;
4286 u64 phys;
4287 u64 len;
4288 u32 flags;
4289 bool cached;
4290};
4291
4292/*
4293 * Helper to submit fiemap extent.
4294 *
4295 * Will try to merge current fiemap extent specified by @offset, @phys,
4296 * @len and @flags with cached one.
4297 * And only when we fails to merge, cached one will be submitted as
4298 * fiemap extent.
4299 *
4300 * Return value is the same as fiemap_fill_next_extent().
4301 */
4302static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
4303 struct fiemap_cache *cache,
4304 u64 offset, u64 phys, u64 len, u32 flags)
4305{
4306 int ret = 0;
4307
4308 if (!cache->cached)
4309 goto assign;
4310
4311 /*
4312 * Sanity check, extent_fiemap() should have ensured that new
Andrea Gelmini52042d82018-11-28 12:05:13 +01004313 * fiemap extent won't overlap with cached one.
Qu Wenruo47518322017-04-07 10:43:15 +08004314 * Not recoverable.
4315 *
4316 * NOTE: Physical address can overlap, due to compression
4317 */
4318 if (cache->offset + cache->len > offset) {
4319 WARN_ON(1);
4320 return -EINVAL;
4321 }
4322
4323 /*
4324 * Only merges fiemap extents if
4325 * 1) Their logical addresses are continuous
4326 *
4327 * 2) Their physical addresses are continuous
4328 * So truly compressed (physical size smaller than logical size)
4329 * extents won't get merged with each other
4330 *
4331 * 3) Share same flags except FIEMAP_EXTENT_LAST
4332 * So regular extent won't get merged with prealloc extent
4333 */
4334 if (cache->offset + cache->len == offset &&
4335 cache->phys + cache->len == phys &&
4336 (cache->flags & ~FIEMAP_EXTENT_LAST) ==
4337 (flags & ~FIEMAP_EXTENT_LAST)) {
4338 cache->len += len;
4339 cache->flags |= flags;
4340 goto try_submit_last;
4341 }
4342
4343 /* Not mergeable, need to submit cached one */
4344 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4345 cache->len, cache->flags);
4346 cache->cached = false;
4347 if (ret)
4348 return ret;
4349assign:
4350 cache->cached = true;
4351 cache->offset = offset;
4352 cache->phys = phys;
4353 cache->len = len;
4354 cache->flags = flags;
4355try_submit_last:
4356 if (cache->flags & FIEMAP_EXTENT_LAST) {
4357 ret = fiemap_fill_next_extent(fieinfo, cache->offset,
4358 cache->phys, cache->len, cache->flags);
4359 cache->cached = false;
4360 }
4361 return ret;
4362}
4363
4364/*
Qu Wenruo848c23b2017-06-22 10:01:21 +08004365 * Emit last fiemap cache
Qu Wenruo47518322017-04-07 10:43:15 +08004366 *
Qu Wenruo848c23b2017-06-22 10:01:21 +08004367 * The last fiemap cache may still be cached in the following case:
4368 * 0 4k 8k
4369 * |<- Fiemap range ->|
4370 * |<------------ First extent ----------->|
4371 *
4372 * In this case, the first extent range will be cached but not emitted.
4373 * So we must emit it before ending extent_fiemap().
Qu Wenruo47518322017-04-07 10:43:15 +08004374 */
Qu Wenruo848c23b2017-06-22 10:01:21 +08004375static int emit_last_fiemap_cache(struct btrfs_fs_info *fs_info,
4376 struct fiemap_extent_info *fieinfo,
4377 struct fiemap_cache *cache)
Qu Wenruo47518322017-04-07 10:43:15 +08004378{
4379 int ret;
4380
4381 if (!cache->cached)
4382 return 0;
4383
Qu Wenruo47518322017-04-07 10:43:15 +08004384 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4385 cache->len, cache->flags);
4386 cache->cached = false;
4387 if (ret > 0)
4388 ret = 0;
4389 return ret;
4390}
4391
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004392int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
David Sterba2135fb92017-06-23 04:09:57 +02004393 __u64 start, __u64 len)
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004394{
Josef Bacik975f84f2010-11-23 19:36:57 +00004395 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004396 u64 off = start;
4397 u64 max = start + len;
4398 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004399 u32 found_type;
4400 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004401 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004402 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004403 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004404 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004405 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004406 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004407 struct btrfs_path *path;
Josef Bacikdc046b12014-09-10 16:20:45 -04004408 struct btrfs_root *root = BTRFS_I(inode)->root;
Qu Wenruo47518322017-04-07 10:43:15 +08004409 struct fiemap_cache cache = { 0 };
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004410 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004411 u64 em_start = 0;
4412 u64 em_len = 0;
4413 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004414
4415 if (len == 0)
4416 return -EINVAL;
4417
Josef Bacik975f84f2010-11-23 19:36:57 +00004418 path = btrfs_alloc_path();
4419 if (!path)
4420 return -ENOMEM;
4421 path->leave_spinning = 1;
4422
Jeff Mahoneyda170662016-06-15 09:22:56 -04004423 start = round_down(start, btrfs_inode_sectorsize(inode));
4424 len = round_up(max, btrfs_inode_sectorsize(inode)) - start;
Josef Bacik4d479cf2011-11-17 11:34:31 -05004425
Chris Masonec29ed52011-02-23 16:23:20 -05004426 /*
4427 * lookup the last file extent. We're not using i_size here
4428 * because there might be preallocation past i_size
4429 */
David Sterbaf85b7372017-01-20 14:54:07 +01004430 ret = btrfs_lookup_file_extent(NULL, root, path,
4431 btrfs_ino(BTRFS_I(inode)), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004432 if (ret < 0) {
4433 btrfs_free_path(path);
4434 return ret;
Liu Bo2d324f52016-05-17 17:21:48 -07004435 } else {
4436 WARN_ON(!ret);
4437 if (ret == 1)
4438 ret = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004439 }
Liu Bo2d324f52016-05-17 17:21:48 -07004440
Josef Bacik975f84f2010-11-23 19:36:57 +00004441 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004442 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
David Sterba962a2982014-06-04 18:41:45 +02004443 found_type = found_key.type;
Josef Bacik975f84f2010-11-23 19:36:57 +00004444
Chris Masonec29ed52011-02-23 16:23:20 -05004445 /* No extents, but there might be delalloc bits */
Nikolay Borisov4a0cc7c2017-01-10 20:35:31 +02004446 if (found_key.objectid != btrfs_ino(BTRFS_I(inode)) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004447 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004448 /* have to trust i_size as the end */
4449 last = (u64)-1;
4450 last_for_get_extent = isize;
4451 } else {
4452 /*
4453 * remember the start of the last extent. There are a
4454 * bunch of different factors that go into the length of the
4455 * extent, so its much less complex to remember where it started
4456 */
4457 last = found_key.offset;
4458 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004459 }
Liu Bofe09e162013-09-22 12:54:23 +08004460 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004461
Chris Masonec29ed52011-02-23 16:23:20 -05004462 /*
4463 * we might have some extents allocated but more delalloc past those
4464 * extents. so, we trust isize unless the start of the last extent is
4465 * beyond isize
4466 */
4467 if (last < isize) {
4468 last = (u64)-1;
4469 last_for_get_extent = isize;
4470 }
4471
David Sterbaff13db42015-12-03 14:30:40 +01004472 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004473 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004474
David Sterbae3350e12017-06-23 04:09:57 +02004475 em = get_extent_skip_holes(inode, start, last_for_get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004476 if (!em)
4477 goto out;
4478 if (IS_ERR(em)) {
4479 ret = PTR_ERR(em);
4480 goto out;
4481 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004482
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004483 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004484 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004485
Chris Masonea8efc72011-03-08 11:54:40 -05004486 /* break if the extent we found is outside the range */
4487 if (em->start >= max || extent_map_end(em) < off)
4488 break;
4489
4490 /*
4491 * get_extent may return an extent that starts before our
4492 * requested range. We have to make sure the ranges
4493 * we return to fiemap always move forward and don't
4494 * overlap, so adjust the offsets here
4495 */
4496 em_start = max(em->start, off);
4497
4498 /*
4499 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004500 * for adjusting the disk offset below. Only do this if the
4501 * extent isn't compressed since our in ram offset may be past
4502 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004503 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004504 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4505 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004506 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004507 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004508 flags = 0;
Filipe Mananaf0986312018-06-20 10:02:30 +01004509 if (em->block_start < EXTENT_MAP_LAST_BYTE)
4510 disko = em->block_start + offset_in_extent;
4511 else
4512 disko = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004513
Chris Masonea8efc72011-03-08 11:54:40 -05004514 /*
4515 * bump off for our next call to get_extent
4516 */
4517 off = extent_map_end(em);
4518 if (off >= max)
4519 end = 1;
4520
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004521 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004522 end = 1;
4523 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004524 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004525 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4526 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004527 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004528 flags |= (FIEMAP_EXTENT_DELALLOC |
4529 FIEMAP_EXTENT_UNKNOWN);
Josef Bacikdc046b12014-09-10 16:20:45 -04004530 } else if (fieinfo->fi_extents_max) {
4531 u64 bytenr = em->block_start -
4532 (em->start - em->orig_start);
Liu Bofe09e162013-09-22 12:54:23 +08004533
Liu Bofe09e162013-09-22 12:54:23 +08004534 /*
4535 * As btrfs supports shared space, this information
4536 * can be exported to userspace tools via
Josef Bacikdc046b12014-09-10 16:20:45 -04004537 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4538 * then we're just getting a count and we can skip the
4539 * lookup stuff.
Liu Bofe09e162013-09-22 12:54:23 +08004540 */
Edmund Nadolskibb739cf2017-06-28 21:56:58 -06004541 ret = btrfs_check_shared(root,
4542 btrfs_ino(BTRFS_I(inode)),
4543 bytenr);
Josef Bacikdc046b12014-09-10 16:20:45 -04004544 if (ret < 0)
Liu Bofe09e162013-09-22 12:54:23 +08004545 goto out_free;
Josef Bacikdc046b12014-09-10 16:20:45 -04004546 if (ret)
Liu Bofe09e162013-09-22 12:54:23 +08004547 flags |= FIEMAP_EXTENT_SHARED;
Josef Bacikdc046b12014-09-10 16:20:45 -04004548 ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004549 }
4550 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4551 flags |= FIEMAP_EXTENT_ENCODED;
Josef Bacik0d2b2372015-05-19 10:44:04 -04004552 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4553 flags |= FIEMAP_EXTENT_UNWRITTEN;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004554
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004555 free_extent_map(em);
4556 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004557 if ((em_start >= last) || em_len == (u64)-1 ||
4558 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004559 flags |= FIEMAP_EXTENT_LAST;
4560 end = 1;
4561 }
4562
Chris Masonec29ed52011-02-23 16:23:20 -05004563 /* now scan forward to see if this is really the last extent. */
David Sterbae3350e12017-06-23 04:09:57 +02004564 em = get_extent_skip_holes(inode, off, last_for_get_extent);
Chris Masonec29ed52011-02-23 16:23:20 -05004565 if (IS_ERR(em)) {
4566 ret = PTR_ERR(em);
4567 goto out;
4568 }
4569 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004570 flags |= FIEMAP_EXTENT_LAST;
4571 end = 1;
4572 }
Qu Wenruo47518322017-04-07 10:43:15 +08004573 ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko,
4574 em_len, flags);
Chengyu Song26e726a2015-03-24 18:12:56 -04004575 if (ret) {
4576 if (ret == 1)
4577 ret = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004578 goto out_free;
Chengyu Song26e726a2015-03-24 18:12:56 -04004579 }
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004580 }
4581out_free:
Qu Wenruo47518322017-04-07 10:43:15 +08004582 if (!ret)
Qu Wenruo848c23b2017-06-22 10:01:21 +08004583 ret = emit_last_fiemap_cache(root->fs_info, fieinfo, &cache);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004584 free_extent_map(em);
4585out:
Liu Bofe09e162013-09-22 12:54:23 +08004586 btrfs_free_path(path);
Liu Boa52f4cd2013-05-01 16:23:41 +00004587 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
David Sterbae43bbe52017-12-12 21:43:52 +01004588 &cached_state);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004589 return ret;
4590}
4591
Chris Mason727011e2010-08-06 13:21:20 -04004592static void __free_extent_buffer(struct extent_buffer *eb)
4593{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004594 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004595 kmem_cache_free(extent_buffer_cache, eb);
4596}
4597
Josef Bacika26e8c92014-03-28 17:07:27 -04004598int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004599{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004600 return (atomic_read(&eb->io_pages) ||
4601 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4602 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004603}
4604
Miao Xie897ca6e92010-10-26 20:57:29 -04004605/*
David Sterba55ac0132018-07-19 17:24:32 +02004606 * Release all pages attached to the extent buffer.
Miao Xie897ca6e92010-10-26 20:57:29 -04004607 */
David Sterba55ac0132018-07-19 17:24:32 +02004608static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
Miao Xie897ca6e92010-10-26 20:57:29 -04004609{
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004610 int i;
4611 int num_pages;
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004612 int mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
Miao Xie897ca6e92010-10-26 20:57:29 -04004613
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004614 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e92010-10-26 20:57:29 -04004615
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004616 num_pages = num_extent_pages(eb);
4617 for (i = 0; i < num_pages; i++) {
4618 struct page *page = eb->pages[i];
Miao Xie897ca6e92010-10-26 20:57:29 -04004619
Forrest Liu5d2361d2015-02-09 17:31:45 +08004620 if (!page)
4621 continue;
4622 if (mapped)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004623 spin_lock(&page->mapping->private_lock);
Forrest Liu5d2361d2015-02-09 17:31:45 +08004624 /*
4625 * We do this since we'll remove the pages after we've
4626 * removed the eb from the radix tree, so we could race
4627 * and have this page now attached to the new eb. So
4628 * only clear page_private if it's still connected to
4629 * this eb.
4630 */
4631 if (PagePrivate(page) &&
4632 page->private == (unsigned long)eb) {
4633 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4634 BUG_ON(PageDirty(page));
4635 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004636 /*
Forrest Liu5d2361d2015-02-09 17:31:45 +08004637 * We need to make sure we haven't be attached
4638 * to a new eb.
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004639 */
Forrest Liu5d2361d2015-02-09 17:31:45 +08004640 ClearPagePrivate(page);
4641 set_page_private(page, 0);
4642 /* One for the page private */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004643 put_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004644 }
Forrest Liu5d2361d2015-02-09 17:31:45 +08004645
4646 if (mapped)
4647 spin_unlock(&page->mapping->private_lock);
4648
Nicholas D Steeves01327612016-05-19 21:18:45 -04004649 /* One for when we allocated the page */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004650 put_page(page);
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004651 }
Miao Xie897ca6e92010-10-26 20:57:29 -04004652}
4653
4654/*
4655 * Helper for releasing the extent buffer.
4656 */
4657static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4658{
David Sterba55ac0132018-07-19 17:24:32 +02004659 btrfs_release_extent_buffer_pages(eb);
Miao Xie897ca6e92010-10-26 20:57:29 -04004660 __free_extent_buffer(eb);
4661}
4662
Josef Bacikf28491e2013-12-16 13:24:27 -05004663static struct extent_buffer *
4664__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
David Sterba23d79d82014-06-15 02:55:29 +02004665 unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004666{
4667 struct extent_buffer *eb = NULL;
4668
Michal Hockod1b5c562015-08-19 14:17:40 +02004669 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004670 eb->start = start;
4671 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004672 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004673 eb->bflags = 0;
4674 rwlock_init(&eb->lock);
4675 atomic_set(&eb->write_locks, 0);
4676 atomic_set(&eb->read_locks, 0);
4677 atomic_set(&eb->blocking_readers, 0);
4678 atomic_set(&eb->blocking_writers, 0);
4679 atomic_set(&eb->spinning_readers, 0);
4680 atomic_set(&eb->spinning_writers, 0);
4681 eb->lock_nested = 0;
4682 init_waitqueue_head(&eb->write_lock_wq);
4683 init_waitqueue_head(&eb->read_lock_wq);
4684
4685 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4686
4687 spin_lock_init(&eb->refs_lock);
4688 atomic_set(&eb->refs, 1);
4689 atomic_set(&eb->io_pages, 0);
4690
4691 /*
4692 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4693 */
4694 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4695 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4696 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4697
4698 return eb;
4699}
4700
4701struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4702{
David Sterbacc5e31a2018-03-01 18:20:27 +01004703 int i;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004704 struct page *p;
4705 struct extent_buffer *new;
David Sterbacc5e31a2018-03-01 18:20:27 +01004706 int num_pages = num_extent_pages(src);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004707
David Sterba3f556f72014-06-15 03:20:26 +02004708 new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004709 if (new == NULL)
4710 return NULL;
4711
4712 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004713 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004714 if (!p) {
4715 btrfs_release_extent_buffer(new);
4716 return NULL;
4717 }
4718 attach_extent_buffer_page(new, p);
4719 WARN_ON(PageDirty(p));
4720 SetPageUptodate(p);
4721 new->pages[i] = p;
David Sterbafba1acf2016-11-08 17:56:24 +01004722 copy_page(page_address(p), page_address(src->pages[i]));
Josef Bacikdb7f3432013-08-07 14:54:37 -04004723 }
4724
Josef Bacikdb7f3432013-08-07 14:54:37 -04004725 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004726 set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004727
4728 return new;
4729}
4730
Omar Sandoval0f331222015-09-29 20:50:31 -07004731struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4732 u64 start, unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004733{
4734 struct extent_buffer *eb;
David Sterbacc5e31a2018-03-01 18:20:27 +01004735 int num_pages;
4736 int i;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004737
David Sterba3f556f72014-06-15 03:20:26 +02004738 eb = __alloc_extent_buffer(fs_info, start, len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004739 if (!eb)
4740 return NULL;
4741
David Sterba65ad0102018-06-29 10:56:49 +02004742 num_pages = num_extent_pages(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004743 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004744 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004745 if (!eb->pages[i])
4746 goto err;
4747 }
4748 set_extent_buffer_uptodate(eb);
4749 btrfs_set_header_nritems(eb, 0);
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004750 set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004751
4752 return eb;
4753err:
4754 for (; i > 0; i--)
4755 __free_page(eb->pages[i - 1]);
4756 __free_extent_buffer(eb);
4757 return NULL;
4758}
4759
Omar Sandoval0f331222015-09-29 20:50:31 -07004760struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
Jeff Mahoneyda170662016-06-15 09:22:56 -04004761 u64 start)
Omar Sandoval0f331222015-09-29 20:50:31 -07004762{
Jeff Mahoneyda170662016-06-15 09:22:56 -04004763 return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
Omar Sandoval0f331222015-09-29 20:50:31 -07004764}
4765
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004766static void check_buffer_tree_ref(struct extent_buffer *eb)
4767{
Chris Mason242e18c2013-01-29 17:49:37 -05004768 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004769 /* the ref bit is tricky. We have to make sure it is set
4770 * if we have the buffer dirty. Otherwise the
4771 * code to free a buffer can end up dropping a dirty
4772 * page
4773 *
4774 * Once the ref bit is set, it won't go away while the
4775 * buffer is dirty or in writeback, and it also won't
4776 * go away while we have the reference count on the
4777 * eb bumped.
4778 *
4779 * We can't just set the ref bit without bumping the
4780 * ref on the eb because free_extent_buffer might
4781 * see the ref bit and try to clear it. If this happens
4782 * free_extent_buffer might end up dropping our original
4783 * ref by mistake and freeing the page before we are able
4784 * to add one more ref.
4785 *
4786 * So bump the ref count first, then set the bit. If someone
4787 * beat us to it, drop the ref we added.
4788 */
Chris Mason242e18c2013-01-29 17:49:37 -05004789 refs = atomic_read(&eb->refs);
4790 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4791 return;
4792
Josef Bacik594831c2012-07-20 16:11:08 -04004793 spin_lock(&eb->refs_lock);
4794 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004795 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004796 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004797}
4798
Mel Gorman2457aec2014-06-04 16:10:31 -07004799static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4800 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04004801{
David Sterbacc5e31a2018-03-01 18:20:27 +01004802 int num_pages, i;
Josef Bacik5df42352012-03-15 18:24:42 -04004803
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004804 check_buffer_tree_ref(eb);
4805
David Sterba65ad0102018-06-29 10:56:49 +02004806 num_pages = num_extent_pages(eb);
Josef Bacik5df42352012-03-15 18:24:42 -04004807 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02004808 struct page *p = eb->pages[i];
4809
Mel Gorman2457aec2014-06-04 16:10:31 -07004810 if (p != accessed)
4811 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004812 }
4813}
4814
Josef Bacikf28491e2013-12-16 13:24:27 -05004815struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4816 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004817{
4818 struct extent_buffer *eb;
4819
4820 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05004821 eb = radix_tree_lookup(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004822 start >> PAGE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004823 if (eb && atomic_inc_not_zero(&eb->refs)) {
4824 rcu_read_unlock();
Filipe Manana062c19e2015-04-23 11:28:48 +01004825 /*
4826 * Lock our eb's refs_lock to avoid races with
4827 * free_extent_buffer. When we get our eb it might be flagged
4828 * with EXTENT_BUFFER_STALE and another task running
4829 * free_extent_buffer might have seen that flag set,
4830 * eb->refs == 2, that the buffer isn't under IO (dirty and
4831 * writeback flags not set) and it's still in the tree (flag
4832 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
4833 * of decrementing the extent buffer's reference count twice.
4834 * So here we could race and increment the eb's reference count,
4835 * clear its stale flag, mark it as dirty and drop our reference
4836 * before the other task finishes executing free_extent_buffer,
4837 * which would later result in an attempt to free an extent
4838 * buffer that is dirty.
4839 */
4840 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
4841 spin_lock(&eb->refs_lock);
4842 spin_unlock(&eb->refs_lock);
4843 }
Mel Gorman2457aec2014-06-04 16:10:31 -07004844 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004845 return eb;
4846 }
4847 rcu_read_unlock();
4848
4849 return NULL;
4850}
4851
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004852#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4853struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
Jeff Mahoneyda170662016-06-15 09:22:56 -04004854 u64 start)
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004855{
4856 struct extent_buffer *eb, *exists = NULL;
4857 int ret;
4858
4859 eb = find_extent_buffer(fs_info, start);
4860 if (eb)
4861 return eb;
Jeff Mahoneyda170662016-06-15 09:22:56 -04004862 eb = alloc_dummy_extent_buffer(fs_info, start);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004863 if (!eb)
4864 return NULL;
4865 eb->fs_info = fs_info;
4866again:
David Sterbae1860a72016-05-09 14:11:38 +02004867 ret = radix_tree_preload(GFP_NOFS);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004868 if (ret)
4869 goto free_eb;
4870 spin_lock(&fs_info->buffer_lock);
4871 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004872 start >> PAGE_SHIFT, eb);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004873 spin_unlock(&fs_info->buffer_lock);
4874 radix_tree_preload_end();
4875 if (ret == -EEXIST) {
4876 exists = find_extent_buffer(fs_info, start);
4877 if (exists)
4878 goto free_eb;
4879 else
4880 goto again;
4881 }
4882 check_buffer_tree_ref(eb);
4883 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4884
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004885 return eb;
4886free_eb:
4887 btrfs_release_extent_buffer(eb);
4888 return exists;
4889}
4890#endif
4891
Josef Bacikf28491e2013-12-16 13:24:27 -05004892struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02004893 u64 start)
Chris Masond1310b22008-01-24 16:13:08 -05004894{
Jeff Mahoneyda170662016-06-15 09:22:56 -04004895 unsigned long len = fs_info->nodesize;
David Sterbacc5e31a2018-03-01 18:20:27 +01004896 int num_pages;
4897 int i;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004898 unsigned long index = start >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05004899 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004900 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004901 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05004902 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05004903 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004904 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004905
Jeff Mahoneyda170662016-06-15 09:22:56 -04004906 if (!IS_ALIGNED(start, fs_info->sectorsize)) {
Liu Boc871b0f2016-06-06 12:01:23 -07004907 btrfs_err(fs_info, "bad tree block start %llu", start);
4908 return ERR_PTR(-EINVAL);
4909 }
4910
Josef Bacikf28491e2013-12-16 13:24:27 -05004911 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004912 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04004913 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004914
David Sterba23d79d82014-06-15 02:55:29 +02004915 eb = __alloc_extent_buffer(fs_info, start, len);
Peter2b114d12008-04-01 11:21:40 -04004916 if (!eb)
Liu Boc871b0f2016-06-06 12:01:23 -07004917 return ERR_PTR(-ENOMEM);
Chris Masond1310b22008-01-24 16:13:08 -05004918
David Sterba65ad0102018-06-29 10:56:49 +02004919 num_pages = num_extent_pages(eb);
Chris Mason727011e2010-08-06 13:21:20 -04004920 for (i = 0; i < num_pages; i++, index++) {
Michal Hockod1b5c562015-08-19 14:17:40 +02004921 p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
Liu Boc871b0f2016-06-06 12:01:23 -07004922 if (!p) {
4923 exists = ERR_PTR(-ENOMEM);
Chris Mason6af118ce2008-07-22 11:18:07 -04004924 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07004925 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004926
4927 spin_lock(&mapping->private_lock);
4928 if (PagePrivate(p)) {
4929 /*
4930 * We could have already allocated an eb for this page
4931 * and attached one so lets see if we can get a ref on
4932 * the existing eb, and if we can we know it's good and
4933 * we can just return that one, else we know we can just
4934 * overwrite page->private.
4935 */
4936 exists = (struct extent_buffer *)p->private;
4937 if (atomic_inc_not_zero(&exists->refs)) {
4938 spin_unlock(&mapping->private_lock);
4939 unlock_page(p);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004940 put_page(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07004941 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004942 goto free_eb;
4943 }
Omar Sandoval5ca64f42015-02-24 02:47:05 -08004944 exists = NULL;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004945
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004946 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004947 * Do this so attach doesn't complain and we need to
4948 * drop the ref the old guy had.
4949 */
4950 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004951 WARN_ON(PageDirty(p));
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004952 put_page(p);
Chris Masond1310b22008-01-24 16:13:08 -05004953 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004954 attach_extent_buffer_page(eb, p);
4955 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004956 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04004957 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004958 if (!PageUptodate(p))
4959 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004960
4961 /*
Nikolay Borisovb16d0112018-07-04 10:24:52 +03004962 * We can't unlock the pages just yet since the extent buffer
4963 * hasn't been properly inserted in the radix tree, this
4964 * opens a race with btree_releasepage which can free a page
4965 * while we are still filling in all pages for the buffer and
4966 * we could crash.
Chris Masoneb14ab82011-02-10 12:35:00 -05004967 */
Chris Masond1310b22008-01-24 16:13:08 -05004968 }
4969 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004970 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004971again:
David Sterbae1860a72016-05-09 14:11:38 +02004972 ret = radix_tree_preload(GFP_NOFS);
Liu Boc871b0f2016-06-06 12:01:23 -07004973 if (ret) {
4974 exists = ERR_PTR(ret);
Miao Xie19fe0a82010-10-26 20:57:29 -04004975 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07004976 }
Miao Xie19fe0a82010-10-26 20:57:29 -04004977
Josef Bacikf28491e2013-12-16 13:24:27 -05004978 spin_lock(&fs_info->buffer_lock);
4979 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004980 start >> PAGE_SHIFT, eb);
Josef Bacikf28491e2013-12-16 13:24:27 -05004981 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004982 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04004983 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004984 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004985 if (exists)
4986 goto free_eb;
4987 else
Josef Bacik115391d2012-03-09 09:51:43 -05004988 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04004989 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004990 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004991 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004992 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05004993
4994 /*
Nikolay Borisovb16d0112018-07-04 10:24:52 +03004995 * Now it's safe to unlock the pages because any calls to
4996 * btree_releasepage will correctly detect that a page belongs to a
4997 * live buffer and won't free them prematurely.
Chris Masoneb14ab82011-02-10 12:35:00 -05004998 */
Nikolay Borisov28187ae2018-07-04 10:24:51 +03004999 for (i = 0; i < num_pages; i++)
5000 unlock_page(eb->pages[i]);
Chris Masond1310b22008-01-24 16:13:08 -05005001 return eb;
5002
Chris Mason6af118ce2008-07-22 11:18:07 -04005003free_eb:
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005004 WARN_ON(!atomic_dec_and_test(&eb->refs));
Chris Mason727011e2010-08-06 13:21:20 -04005005 for (i = 0; i < num_pages; i++) {
5006 if (eb->pages[i])
5007 unlock_page(eb->pages[i]);
5008 }
Chris Masoneb14ab82011-02-10 12:35:00 -05005009
Miao Xie897ca6e92010-10-26 20:57:29 -04005010 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005011 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05005012}
Chris Masond1310b22008-01-24 16:13:08 -05005013
Josef Bacik3083ee22012-03-09 16:01:49 -05005014static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
5015{
5016 struct extent_buffer *eb =
5017 container_of(head, struct extent_buffer, rcu_head);
5018
5019 __free_extent_buffer(eb);
5020}
5021
David Sterbaf7a52a42013-04-26 14:56:29 +00005022static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05005023{
Nikolay Borisov07e21c42018-06-27 16:38:23 +03005024 lockdep_assert_held(&eb->refs_lock);
5025
Josef Bacik3083ee22012-03-09 16:01:49 -05005026 WARN_ON(atomic_read(&eb->refs) == 0);
5027 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05005028 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005029 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05005030
Jan Schmidt815a51c2012-05-16 17:00:02 +02005031 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05005032
Josef Bacikf28491e2013-12-16 13:24:27 -05005033 spin_lock(&fs_info->buffer_lock);
5034 radix_tree_delete(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005035 eb->start >> PAGE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05005036 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005037 } else {
5038 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02005039 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005040
5041 /* Should be safe to release our pages at this point */
David Sterba55ac0132018-07-19 17:24:32 +02005042 btrfs_release_extent_buffer_pages(eb);
Josef Bacikbcb7e442015-03-16 17:38:02 -04005043#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
Nikolay Borisovb0132a32018-06-27 16:38:24 +03005044 if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
Josef Bacikbcb7e442015-03-16 17:38:02 -04005045 __free_extent_buffer(eb);
5046 return 1;
5047 }
5048#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05005049 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04005050 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05005051 }
5052 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04005053
5054 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05005055}
5056
Chris Masond1310b22008-01-24 16:13:08 -05005057void free_extent_buffer(struct extent_buffer *eb)
5058{
Chris Mason242e18c2013-01-29 17:49:37 -05005059 int refs;
5060 int old;
Chris Masond1310b22008-01-24 16:13:08 -05005061 if (!eb)
5062 return;
5063
Chris Mason242e18c2013-01-29 17:49:37 -05005064 while (1) {
5065 refs = atomic_read(&eb->refs);
Nikolay Borisov46cc7752018-10-15 17:04:01 +03005066 if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
5067 || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
5068 refs == 1))
Chris Mason242e18c2013-01-29 17:49:37 -05005069 break;
5070 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5071 if (old == refs)
5072 return;
5073 }
5074
Josef Bacik3083ee22012-03-09 16:01:49 -05005075 spin_lock(&eb->refs_lock);
5076 if (atomic_read(&eb->refs) == 2 &&
5077 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005078 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005079 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5080 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05005081
Josef Bacik3083ee22012-03-09 16:01:49 -05005082 /*
5083 * I know this is terrible, but it's temporary until we stop tracking
5084 * the uptodate bits and such for the extent buffers.
5085 */
David Sterbaf7a52a42013-04-26 14:56:29 +00005086 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005087}
Chris Masond1310b22008-01-24 16:13:08 -05005088
Josef Bacik3083ee22012-03-09 16:01:49 -05005089void free_extent_buffer_stale(struct extent_buffer *eb)
5090{
5091 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05005092 return;
5093
Josef Bacik3083ee22012-03-09 16:01:49 -05005094 spin_lock(&eb->refs_lock);
5095 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5096
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005097 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005098 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5099 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00005100 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005101}
5102
Chris Mason1d4284b2012-03-28 20:31:37 -04005103void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005104{
David Sterbacc5e31a2018-03-01 18:20:27 +01005105 int i;
5106 int num_pages;
Chris Masond1310b22008-01-24 16:13:08 -05005107 struct page *page;
5108
David Sterba65ad0102018-06-29 10:56:49 +02005109 num_pages = num_extent_pages(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005110
5111 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005112 page = eb->pages[i];
Chris Masonb9473432009-03-13 11:00:37 -04005113 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05005114 continue;
5115
Chris Masona61e6f22008-07-22 11:18:08 -04005116 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05005117 WARN_ON(!PagePrivate(page));
5118
Chris Masond1310b22008-01-24 16:13:08 -05005119 clear_page_dirty_for_io(page);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07005120 xa_lock_irq(&page->mapping->i_pages);
Matthew Wilcox0a943c62017-12-04 10:37:22 -05005121 if (!PageDirty(page))
5122 __xa_clear_mark(&page->mapping->i_pages,
5123 page_index(page), PAGECACHE_TAG_DIRTY);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07005124 xa_unlock_irq(&page->mapping->i_pages);
Chris Masonbf0da8c2011-11-04 12:29:37 -04005125 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04005126 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05005127 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005128 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05005129}
Chris Masond1310b22008-01-24 16:13:08 -05005130
Liu Boabb57ef2018-09-14 01:44:42 +08005131bool set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005132{
David Sterbacc5e31a2018-03-01 18:20:27 +01005133 int i;
5134 int num_pages;
Liu Boabb57ef2018-09-14 01:44:42 +08005135 bool was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005136
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005137 check_buffer_tree_ref(eb);
5138
Chris Masonb9473432009-03-13 11:00:37 -04005139 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005140
David Sterba65ad0102018-06-29 10:56:49 +02005141 num_pages = num_extent_pages(eb);
Josef Bacik3083ee22012-03-09 16:01:49 -05005142 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005143 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5144
Liu Boabb57ef2018-09-14 01:44:42 +08005145 if (!was_dirty)
5146 for (i = 0; i < num_pages; i++)
5147 set_page_dirty(eb->pages[i]);
Liu Bo51995c32018-09-14 01:46:08 +08005148
5149#ifdef CONFIG_BTRFS_DEBUG
5150 for (i = 0; i < num_pages; i++)
5151 ASSERT(PageDirty(eb->pages[i]));
5152#endif
5153
Chris Masonb9473432009-03-13 11:00:37 -04005154 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005155}
Chris Masond1310b22008-01-24 16:13:08 -05005156
David Sterba69ba3922015-12-03 13:08:59 +01005157void clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04005158{
David Sterbacc5e31a2018-03-01 18:20:27 +01005159 int i;
Chris Mason1259ab72008-05-12 13:39:03 -04005160 struct page *page;
David Sterbacc5e31a2018-03-01 18:20:27 +01005161 int num_pages;
Chris Mason1259ab72008-05-12 13:39:03 -04005162
Chris Masonb4ce94d2009-02-04 09:25:08 -05005163 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02005164 num_pages = num_extent_pages(eb);
Chris Mason1259ab72008-05-12 13:39:03 -04005165 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005166 page = eb->pages[i];
Chris Mason33958dc2008-07-30 10:29:12 -04005167 if (page)
5168 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04005169 }
Chris Mason1259ab72008-05-12 13:39:03 -04005170}
5171
David Sterba09c25a82015-12-03 13:08:59 +01005172void set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005173{
David Sterbacc5e31a2018-03-01 18:20:27 +01005174 int i;
Chris Masond1310b22008-01-24 16:13:08 -05005175 struct page *page;
David Sterbacc5e31a2018-03-01 18:20:27 +01005176 int num_pages;
Chris Masond1310b22008-01-24 16:13:08 -05005177
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005178 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02005179 num_pages = num_extent_pages(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005180 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005181 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005182 SetPageUptodate(page);
5183 }
Chris Masond1310b22008-01-24 16:13:08 -05005184}
Chris Masond1310b22008-01-24 16:13:08 -05005185
Chris Masond1310b22008-01-24 16:13:08 -05005186int read_extent_buffer_pages(struct extent_io_tree *tree,
David Sterba6af49db2017-06-23 04:09:57 +02005187 struct extent_buffer *eb, int wait, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05005188{
David Sterbacc5e31a2018-03-01 18:20:27 +01005189 int i;
Chris Masond1310b22008-01-24 16:13:08 -05005190 struct page *page;
5191 int err;
5192 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04005193 int locked_pages = 0;
5194 int all_uptodate = 1;
David Sterbacc5e31a2018-03-01 18:20:27 +01005195 int num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04005196 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005197 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04005198 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005199
Chris Masonb4ce94d2009-02-04 09:25:08 -05005200 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05005201 return 0;
5202
David Sterba65ad0102018-06-29 10:56:49 +02005203 num_pages = num_extent_pages(eb);
Josef Bacik8436ea912016-09-02 15:40:03 -04005204 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005205 page = eb->pages[i];
Arne Jansenbb82ab82011-06-10 14:06:53 +02005206 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04005207 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04005208 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05005209 } else {
5210 lock_page(page);
5211 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005212 locked_pages++;
Liu Bo2571e732016-08-03 12:33:01 -07005213 }
5214 /*
5215 * We need to firstly lock all pages to make sure that
5216 * the uptodate bit of our pages won't be affected by
5217 * clear_extent_buffer_uptodate().
5218 */
Josef Bacik8436ea912016-09-02 15:40:03 -04005219 for (i = 0; i < num_pages; i++) {
Liu Bo2571e732016-08-03 12:33:01 -07005220 page = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04005221 if (!PageUptodate(page)) {
5222 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04005223 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04005224 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005225 }
Liu Bo2571e732016-08-03 12:33:01 -07005226
Chris Masonce9adaa2008-04-09 16:28:12 -04005227 if (all_uptodate) {
Josef Bacik8436ea912016-09-02 15:40:03 -04005228 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04005229 goto unlock_exit;
5230 }
5231
Filipe Manana656f30d2014-09-26 12:25:56 +01005232 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04005233 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005234 atomic_set(&eb->io_pages, num_reads);
Josef Bacik8436ea912016-09-02 15:40:03 -04005235 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005236 page = eb->pages[i];
Liu Bobaf863b2016-07-11 10:39:07 -07005237
Chris Masonce9adaa2008-04-09 16:28:12 -04005238 if (!PageUptodate(page)) {
Liu Bobaf863b2016-07-11 10:39:07 -07005239 if (ret) {
5240 atomic_dec(&eb->io_pages);
5241 unlock_page(page);
5242 continue;
5243 }
5244
Chris Masonf1885912008-04-09 16:28:12 -04005245 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05005246 err = __extent_read_full_page(tree, page,
David Sterba6af49db2017-06-23 04:09:57 +02005247 btree_get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005248 mirror_num, &bio_flags,
Mike Christie1f7ad752016-06-05 14:31:51 -05005249 REQ_META);
Liu Bobaf863b2016-07-11 10:39:07 -07005250 if (err) {
Chris Masond1310b22008-01-24 16:13:08 -05005251 ret = err;
Liu Bobaf863b2016-07-11 10:39:07 -07005252 /*
5253 * We use &bio in above __extent_read_full_page,
5254 * so we ensure that if it returns error, the
5255 * current page fails to add itself to bio and
5256 * it's been unlocked.
5257 *
5258 * We must dec io_pages by ourselves.
5259 */
5260 atomic_dec(&eb->io_pages);
5261 }
Chris Masond1310b22008-01-24 16:13:08 -05005262 } else {
5263 unlock_page(page);
5264 }
5265 }
5266
Jeff Mahoney355808c2011-10-03 23:23:14 -04005267 if (bio) {
Mike Christie1f7ad752016-06-05 14:31:51 -05005268 err = submit_one_bio(bio, mirror_num, bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005269 if (err)
5270 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005271 }
Chris Masona86c12c2008-02-07 10:50:54 -05005272
Arne Jansenbb82ab82011-06-10 14:06:53 +02005273 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005274 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005275
Josef Bacik8436ea912016-09-02 15:40:03 -04005276 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005277 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005278 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005279 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005280 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005281 }
Chris Masond3977122009-01-05 21:25:51 -05005282
Chris Masond1310b22008-01-24 16:13:08 -05005283 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005284
5285unlock_exit:
Chris Masond3977122009-01-05 21:25:51 -05005286 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04005287 locked_pages--;
Josef Bacik8436ea912016-09-02 15:40:03 -04005288 page = eb->pages[locked_pages];
5289 unlock_page(page);
Chris Masonce9adaa2008-04-09 16:28:12 -04005290 }
5291 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005292}
Chris Masond1310b22008-01-24 16:13:08 -05005293
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005294void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
5295 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005296{
5297 size_t cur;
5298 size_t offset;
5299 struct page *page;
5300 char *kaddr;
5301 char *dst = (char *)dstv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005302 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005303 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005304
Liu Bof716abd2017-08-09 11:10:16 -06005305 if (start + len > eb->len) {
5306 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5307 eb->start, eb->len, start, len);
5308 memset(dst, 0, len);
5309 return;
5310 }
Chris Masond1310b22008-01-24 16:13:08 -05005311
Johannes Thumshirn70730172018-12-05 15:23:03 +01005312 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005313
Chris Masond3977122009-01-05 21:25:51 -05005314 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005315 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005316
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005317 cur = min(len, (PAGE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005318 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005319 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005320
5321 dst += cur;
5322 len -= cur;
5323 offset = 0;
5324 i++;
5325 }
5326}
Chris Masond1310b22008-01-24 16:13:08 -05005327
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005328int read_extent_buffer_to_user(const struct extent_buffer *eb,
5329 void __user *dstv,
5330 unsigned long start, unsigned long len)
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005331{
5332 size_t cur;
5333 size_t offset;
5334 struct page *page;
5335 char *kaddr;
5336 char __user *dst = (char __user *)dstv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005337 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005338 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005339 int ret = 0;
5340
5341 WARN_ON(start > eb->len);
5342 WARN_ON(start + len > eb->start + eb->len);
5343
Johannes Thumshirn70730172018-12-05 15:23:03 +01005344 offset = offset_in_page(start_offset + start);
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005345
5346 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005347 page = eb->pages[i];
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005348
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005349 cur = min(len, (PAGE_SIZE - offset));
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005350 kaddr = page_address(page);
5351 if (copy_to_user(dst, kaddr + offset, cur)) {
5352 ret = -EFAULT;
5353 break;
5354 }
5355
5356 dst += cur;
5357 len -= cur;
5358 offset = 0;
5359 i++;
5360 }
5361
5362 return ret;
5363}
5364
Liu Bo415b35a2016-06-17 19:16:21 -07005365/*
5366 * return 0 if the item is found within a page.
5367 * return 1 if the item spans two pages.
5368 * return -EINVAL otherwise.
5369 */
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005370int map_private_extent_buffer(const struct extent_buffer *eb,
5371 unsigned long start, unsigned long min_len,
5372 char **map, unsigned long *map_start,
5373 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005374{
Johannes Thumshirncc2c39d2018-11-28 09:54:54 +01005375 size_t offset;
Chris Masond1310b22008-01-24 16:13:08 -05005376 char *kaddr;
5377 struct page *p;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005378 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005379 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005380 unsigned long end_i = (start_offset + start + min_len - 1) >>
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005381 PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005382
Liu Bof716abd2017-08-09 11:10:16 -06005383 if (start + min_len > eb->len) {
5384 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5385 eb->start, eb->len, start, min_len);
5386 return -EINVAL;
5387 }
5388
Chris Masond1310b22008-01-24 16:13:08 -05005389 if (i != end_i)
Liu Bo415b35a2016-06-17 19:16:21 -07005390 return 1;
Chris Masond1310b22008-01-24 16:13:08 -05005391
5392 if (i == 0) {
5393 offset = start_offset;
5394 *map_start = 0;
5395 } else {
5396 offset = 0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005397 *map_start = ((u64)i << PAGE_SHIFT) - start_offset;
Chris Masond1310b22008-01-24 16:13:08 -05005398 }
Chris Masond3977122009-01-05 21:25:51 -05005399
David Sterbafb85fc92014-07-31 01:03:53 +02005400 p = eb->pages[i];
Chris Masona6591712011-07-19 12:04:14 -04005401 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005402 *map = kaddr + offset;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005403 *map_len = PAGE_SIZE - offset;
Chris Masond1310b22008-01-24 16:13:08 -05005404 return 0;
5405}
Chris Masond1310b22008-01-24 16:13:08 -05005406
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005407int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
5408 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005409{
5410 size_t cur;
5411 size_t offset;
5412 struct page *page;
5413 char *kaddr;
5414 char *ptr = (char *)ptrv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005415 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005416 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005417 int ret = 0;
5418
5419 WARN_ON(start > eb->len);
5420 WARN_ON(start + len > eb->start + eb->len);
5421
Johannes Thumshirn70730172018-12-05 15:23:03 +01005422 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005423
Chris Masond3977122009-01-05 21:25:51 -05005424 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005425 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005426
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005427 cur = min(len, (PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005428
Chris Masona6591712011-07-19 12:04:14 -04005429 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005430 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005431 if (ret)
5432 break;
5433
5434 ptr += cur;
5435 len -= cur;
5436 offset = 0;
5437 i++;
5438 }
5439 return ret;
5440}
Chris Masond1310b22008-01-24 16:13:08 -05005441
David Sterbaf157bf72016-11-09 17:43:38 +01005442void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb,
5443 const void *srcv)
5444{
5445 char *kaddr;
5446
5447 WARN_ON(!PageUptodate(eb->pages[0]));
5448 kaddr = page_address(eb->pages[0]);
5449 memcpy(kaddr + offsetof(struct btrfs_header, chunk_tree_uuid), srcv,
5450 BTRFS_FSID_SIZE);
5451}
5452
5453void write_extent_buffer_fsid(struct extent_buffer *eb, const void *srcv)
5454{
5455 char *kaddr;
5456
5457 WARN_ON(!PageUptodate(eb->pages[0]));
5458 kaddr = page_address(eb->pages[0]);
5459 memcpy(kaddr + offsetof(struct btrfs_header, fsid), srcv,
5460 BTRFS_FSID_SIZE);
5461}
5462
Chris Masond1310b22008-01-24 16:13:08 -05005463void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5464 unsigned long start, unsigned long len)
5465{
5466 size_t cur;
5467 size_t offset;
5468 struct page *page;
5469 char *kaddr;
5470 char *src = (char *)srcv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005471 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005472 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005473
5474 WARN_ON(start > eb->len);
5475 WARN_ON(start + len > eb->start + eb->len);
5476
Johannes Thumshirn70730172018-12-05 15:23:03 +01005477 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005478
Chris Masond3977122009-01-05 21:25:51 -05005479 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005480 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005481 WARN_ON(!PageUptodate(page));
5482
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005483 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005484 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005485 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005486
5487 src += cur;
5488 len -= cur;
5489 offset = 0;
5490 i++;
5491 }
5492}
Chris Masond1310b22008-01-24 16:13:08 -05005493
David Sterbab159fa22016-11-08 18:09:03 +01005494void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start,
5495 unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005496{
5497 size_t cur;
5498 size_t offset;
5499 struct page *page;
5500 char *kaddr;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005501 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005502 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005503
5504 WARN_ON(start > eb->len);
5505 WARN_ON(start + len > eb->start + eb->len);
5506
Johannes Thumshirn70730172018-12-05 15:23:03 +01005507 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005508
Chris Masond3977122009-01-05 21:25:51 -05005509 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005510 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005511 WARN_ON(!PageUptodate(page));
5512
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005513 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005514 kaddr = page_address(page);
David Sterbab159fa22016-11-08 18:09:03 +01005515 memset(kaddr + offset, 0, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005516
5517 len -= cur;
5518 offset = 0;
5519 i++;
5520 }
5521}
Chris Masond1310b22008-01-24 16:13:08 -05005522
David Sterba58e80122016-11-08 18:30:31 +01005523void copy_extent_buffer_full(struct extent_buffer *dst,
5524 struct extent_buffer *src)
5525{
5526 int i;
David Sterbacc5e31a2018-03-01 18:20:27 +01005527 int num_pages;
David Sterba58e80122016-11-08 18:30:31 +01005528
5529 ASSERT(dst->len == src->len);
5530
David Sterba65ad0102018-06-29 10:56:49 +02005531 num_pages = num_extent_pages(dst);
David Sterba58e80122016-11-08 18:30:31 +01005532 for (i = 0; i < num_pages; i++)
5533 copy_page(page_address(dst->pages[i]),
5534 page_address(src->pages[i]));
5535}
5536
Chris Masond1310b22008-01-24 16:13:08 -05005537void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5538 unsigned long dst_offset, unsigned long src_offset,
5539 unsigned long len)
5540{
5541 u64 dst_len = dst->len;
5542 size_t cur;
5543 size_t offset;
5544 struct page *page;
5545 char *kaddr;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005546 size_t start_offset = offset_in_page(dst->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005547 unsigned long i = (start_offset + dst_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005548
5549 WARN_ON(src->len != dst_len);
5550
Johannes Thumshirn70730172018-12-05 15:23:03 +01005551 offset = offset_in_page(start_offset + dst_offset);
Chris Masond1310b22008-01-24 16:13:08 -05005552
Chris Masond3977122009-01-05 21:25:51 -05005553 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005554 page = dst->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005555 WARN_ON(!PageUptodate(page));
5556
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005557 cur = min(len, (unsigned long)(PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005558
Chris Masona6591712011-07-19 12:04:14 -04005559 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005560 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005561
5562 src_offset += cur;
5563 len -= cur;
5564 offset = 0;
5565 i++;
5566 }
5567}
Chris Masond1310b22008-01-24 16:13:08 -05005568
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005569/*
5570 * eb_bitmap_offset() - calculate the page and offset of the byte containing the
5571 * given bit number
5572 * @eb: the extent buffer
5573 * @start: offset of the bitmap item in the extent buffer
5574 * @nr: bit number
5575 * @page_index: return index of the page in the extent buffer that contains the
5576 * given bit number
5577 * @page_offset: return offset into the page given by page_index
5578 *
5579 * This helper hides the ugliness of finding the byte in an extent buffer which
5580 * contains a given bit.
5581 */
5582static inline void eb_bitmap_offset(struct extent_buffer *eb,
5583 unsigned long start, unsigned long nr,
5584 unsigned long *page_index,
5585 size_t *page_offset)
5586{
Johannes Thumshirn70730172018-12-05 15:23:03 +01005587 size_t start_offset = offset_in_page(eb->start);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005588 size_t byte_offset = BIT_BYTE(nr);
5589 size_t offset;
5590
5591 /*
5592 * The byte we want is the offset of the extent buffer + the offset of
5593 * the bitmap item in the extent buffer + the offset of the byte in the
5594 * bitmap item.
5595 */
5596 offset = start_offset + start + byte_offset;
5597
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005598 *page_index = offset >> PAGE_SHIFT;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005599 *page_offset = offset_in_page(offset);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005600}
5601
5602/**
5603 * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
5604 * @eb: the extent buffer
5605 * @start: offset of the bitmap item in the extent buffer
5606 * @nr: bit number to test
5607 */
5608int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start,
5609 unsigned long nr)
5610{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005611 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005612 struct page *page;
5613 unsigned long i;
5614 size_t offset;
5615
5616 eb_bitmap_offset(eb, start, nr, &i, &offset);
5617 page = eb->pages[i];
5618 WARN_ON(!PageUptodate(page));
5619 kaddr = page_address(page);
5620 return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
5621}
5622
5623/**
5624 * extent_buffer_bitmap_set - set an area of a bitmap
5625 * @eb: the extent buffer
5626 * @start: offset of the bitmap item in the extent buffer
5627 * @pos: bit number of the first bit
5628 * @len: number of bits to set
5629 */
5630void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start,
5631 unsigned long pos, unsigned long len)
5632{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005633 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005634 struct page *page;
5635 unsigned long i;
5636 size_t offset;
5637 const unsigned int size = pos + len;
5638 int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005639 u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005640
5641 eb_bitmap_offset(eb, start, pos, &i, &offset);
5642 page = eb->pages[i];
5643 WARN_ON(!PageUptodate(page));
5644 kaddr = page_address(page);
5645
5646 while (len >= bits_to_set) {
5647 kaddr[offset] |= mask_to_set;
5648 len -= bits_to_set;
5649 bits_to_set = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005650 mask_to_set = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005651 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005652 offset = 0;
5653 page = eb->pages[++i];
5654 WARN_ON(!PageUptodate(page));
5655 kaddr = page_address(page);
5656 }
5657 }
5658 if (len) {
5659 mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
5660 kaddr[offset] |= mask_to_set;
5661 }
5662}
5663
5664
5665/**
5666 * extent_buffer_bitmap_clear - clear an area of a bitmap
5667 * @eb: the extent buffer
5668 * @start: offset of the bitmap item in the extent buffer
5669 * @pos: bit number of the first bit
5670 * @len: number of bits to clear
5671 */
5672void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start,
5673 unsigned long pos, unsigned long len)
5674{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005675 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005676 struct page *page;
5677 unsigned long i;
5678 size_t offset;
5679 const unsigned int size = pos + len;
5680 int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005681 u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005682
5683 eb_bitmap_offset(eb, start, pos, &i, &offset);
5684 page = eb->pages[i];
5685 WARN_ON(!PageUptodate(page));
5686 kaddr = page_address(page);
5687
5688 while (len >= bits_to_clear) {
5689 kaddr[offset] &= ~mask_to_clear;
5690 len -= bits_to_clear;
5691 bits_to_clear = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005692 mask_to_clear = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005693 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005694 offset = 0;
5695 page = eb->pages[++i];
5696 WARN_ON(!PageUptodate(page));
5697 kaddr = page_address(page);
5698 }
5699 }
5700 if (len) {
5701 mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
5702 kaddr[offset] &= ~mask_to_clear;
5703 }
5704}
5705
Sergei Trofimovich33872062011-04-11 21:52:52 +00005706static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5707{
5708 unsigned long distance = (src > dst) ? src - dst : dst - src;
5709 return distance < len;
5710}
5711
Chris Masond1310b22008-01-24 16:13:08 -05005712static void copy_pages(struct page *dst_page, struct page *src_page,
5713 unsigned long dst_off, unsigned long src_off,
5714 unsigned long len)
5715{
Chris Masona6591712011-07-19 12:04:14 -04005716 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005717 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005718 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005719
Sergei Trofimovich33872062011-04-11 21:52:52 +00005720 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005721 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005722 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005723 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005724 if (areas_overlap(src_off, dst_off, len))
5725 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005726 }
Chris Masond1310b22008-01-24 16:13:08 -05005727
Chris Mason727011e2010-08-06 13:21:20 -04005728 if (must_memmove)
5729 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5730 else
5731 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005732}
5733
5734void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5735 unsigned long src_offset, unsigned long len)
5736{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005737 struct btrfs_fs_info *fs_info = dst->fs_info;
Chris Masond1310b22008-01-24 16:13:08 -05005738 size_t cur;
5739 size_t dst_off_in_page;
5740 size_t src_off_in_page;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005741 size_t start_offset = offset_in_page(dst->start);
Chris Masond1310b22008-01-24 16:13:08 -05005742 unsigned long dst_i;
5743 unsigned long src_i;
5744
5745 if (src_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005746 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005747 "memmove bogus src_offset %lu move len %lu dst len %lu",
5748 src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005749 BUG_ON(1);
5750 }
5751 if (dst_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005752 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005753 "memmove bogus dst_offset %lu move len %lu dst len %lu",
5754 dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005755 BUG_ON(1);
5756 }
5757
Chris Masond3977122009-01-05 21:25:51 -05005758 while (len > 0) {
Johannes Thumshirn70730172018-12-05 15:23:03 +01005759 dst_off_in_page = offset_in_page(start_offset + dst_offset);
5760 src_off_in_page = offset_in_page(start_offset + src_offset);
Chris Masond1310b22008-01-24 16:13:08 -05005761
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005762 dst_i = (start_offset + dst_offset) >> PAGE_SHIFT;
5763 src_i = (start_offset + src_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005764
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005765 cur = min(len, (unsigned long)(PAGE_SIZE -
Chris Masond1310b22008-01-24 16:13:08 -05005766 src_off_in_page));
5767 cur = min_t(unsigned long, cur,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005768 (unsigned long)(PAGE_SIZE - dst_off_in_page));
Chris Masond1310b22008-01-24 16:13:08 -05005769
David Sterbafb85fc92014-07-31 01:03:53 +02005770 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005771 dst_off_in_page, src_off_in_page, cur);
5772
5773 src_offset += cur;
5774 dst_offset += cur;
5775 len -= cur;
5776 }
5777}
Chris Masond1310b22008-01-24 16:13:08 -05005778
5779void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5780 unsigned long src_offset, unsigned long len)
5781{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005782 struct btrfs_fs_info *fs_info = dst->fs_info;
Chris Masond1310b22008-01-24 16:13:08 -05005783 size_t cur;
5784 size_t dst_off_in_page;
5785 size_t src_off_in_page;
5786 unsigned long dst_end = dst_offset + len - 1;
5787 unsigned long src_end = src_offset + len - 1;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005788 size_t start_offset = offset_in_page(dst->start);
Chris Masond1310b22008-01-24 16:13:08 -05005789 unsigned long dst_i;
5790 unsigned long src_i;
5791
5792 if (src_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005793 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005794 "memmove bogus src_offset %lu move len %lu len %lu",
5795 src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005796 BUG_ON(1);
5797 }
5798 if (dst_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005799 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005800 "memmove bogus dst_offset %lu move len %lu len %lu",
5801 dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005802 BUG_ON(1);
5803 }
Chris Mason727011e2010-08-06 13:21:20 -04005804 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005805 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5806 return;
5807 }
Chris Masond3977122009-01-05 21:25:51 -05005808 while (len > 0) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005809 dst_i = (start_offset + dst_end) >> PAGE_SHIFT;
5810 src_i = (start_offset + src_end) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005811
Johannes Thumshirn70730172018-12-05 15:23:03 +01005812 dst_off_in_page = offset_in_page(start_offset + dst_end);
5813 src_off_in_page = offset_in_page(start_offset + src_end);
Chris Masond1310b22008-01-24 16:13:08 -05005814
5815 cur = min_t(unsigned long, len, src_off_in_page + 1);
5816 cur = min(cur, dst_off_in_page + 1);
David Sterbafb85fc92014-07-31 01:03:53 +02005817 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005818 dst_off_in_page - cur + 1,
5819 src_off_in_page - cur + 1, cur);
5820
5821 dst_end -= cur;
5822 src_end -= cur;
5823 len -= cur;
5824 }
5825}
Chris Mason6af118ce2008-07-22 11:18:07 -04005826
David Sterbaf7a52a42013-04-26 14:56:29 +00005827int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005828{
Chris Mason6af118ce2008-07-22 11:18:07 -04005829 struct extent_buffer *eb;
Miao Xie897ca6e92010-10-26 20:57:29 -04005830
Miao Xie19fe0a82010-10-26 20:57:29 -04005831 /*
Nicholas D Steeves01327612016-05-19 21:18:45 -04005832 * We need to make sure nobody is attaching this page to an eb right
Josef Bacik3083ee22012-03-09 16:01:49 -05005833 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005834 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005835 spin_lock(&page->mapping->private_lock);
5836 if (!PagePrivate(page)) {
5837 spin_unlock(&page->mapping->private_lock);
5838 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005839 }
5840
Josef Bacik3083ee22012-03-09 16:01:49 -05005841 eb = (struct extent_buffer *)page->private;
5842 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005843
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005844 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005845 * This is a little awful but should be ok, we need to make sure that
5846 * the eb doesn't disappear out from under us while we're looking at
5847 * this page.
5848 */
5849 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005850 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05005851 spin_unlock(&eb->refs_lock);
5852 spin_unlock(&page->mapping->private_lock);
5853 return 0;
5854 }
5855 spin_unlock(&page->mapping->private_lock);
5856
Josef Bacik3083ee22012-03-09 16:01:49 -05005857 /*
5858 * If tree ref isn't set then we know the ref on this eb is a real ref,
5859 * so just return, this page will likely be freed soon anyway.
5860 */
5861 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5862 spin_unlock(&eb->refs_lock);
5863 return 0;
5864 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005865
David Sterbaf7a52a42013-04-26 14:56:29 +00005866 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005867}