blob: 7a2f657f5dbdcad4f9998252dd9bbce579a72378 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
David Sterbac1d7c512018-04-03 19:23:33 +02002
Chris Masond1310b22008-01-24 16:13:08 -05003#include <linux/bitops.h>
4#include <linux/slab.h>
5#include <linux/bio.h>
6#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/pagemap.h>
8#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05009#include <linux/spinlock.h>
10#include <linux/blkdev.h>
11#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050012#include <linux/writeback.h>
13#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070014#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060015#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050016#include "extent_io.h"
Josef Bacik9c7d3a52019-09-23 10:05:19 -040017#include "extent-io-tree.h"
Chris Masond1310b22008-01-24 16:13:08 -050018#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040019#include "ctree.h"
20#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020021#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010022#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040023#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040024#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080025#include "backref.h"
David Sterba6af49db2017-06-23 04:09:57 +020026#include "disk-io.h"
Chris Masond1310b22008-01-24 16:13:08 -050027
Chris Masond1310b22008-01-24 16:13:08 -050028static struct kmem_cache *extent_state_cache;
29static struct kmem_cache *extent_buffer_cache;
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -040030static struct bio_set btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050031
Filipe Manana27a35072014-07-06 20:09:59 +010032static inline bool extent_state_in_tree(const struct extent_state *state)
33{
34 return !RB_EMPTY_NODE(&state->rb_node);
35}
36
Eric Sandeen6d49ba12013-04-22 16:12:31 +000037#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050038static LIST_HEAD(buffers);
39static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040040
Chris Masond3977122009-01-05 21:25:51 -050041static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000042
43static inline
44void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
45{
46 unsigned long flags;
47
48 spin_lock_irqsave(&leak_lock, flags);
49 list_add(new, head);
50 spin_unlock_irqrestore(&leak_lock, flags);
51}
52
53static inline
54void btrfs_leak_debug_del(struct list_head *entry)
55{
56 unsigned long flags;
57
58 spin_lock_irqsave(&leak_lock, flags);
59 list_del(entry);
60 spin_unlock_irqrestore(&leak_lock, flags);
61}
62
Josef Bacik33ca8322019-09-23 10:05:17 -040063static inline void btrfs_extent_buffer_leak_debug_check(void)
64{
65 struct extent_buffer *eb;
66
67 while (!list_empty(&buffers)) {
68 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
69 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
70 eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
71 list_del(&eb->leak_list);
72 kmem_cache_free(extent_buffer_cache, eb);
73 }
74}
75
76static inline void btrfs_extent_state_leak_debug_check(void)
Eric Sandeen6d49ba12013-04-22 16:12:31 +000077{
78 struct extent_state *state;
Eric Sandeen6d49ba12013-04-22 16:12:31 +000079
80 while (!list_empty(&states)) {
81 state = list_entry(states.next, struct extent_state, leak_list);
David Sterba9ee49a042015-01-14 19:52:13 +010082 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
Filipe Manana27a35072014-07-06 20:09:59 +010083 state->start, state->end, state->state,
84 extent_state_in_tree(state),
Elena Reshetovab7ac31b2017-03-03 10:55:19 +020085 refcount_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000086 list_del(&state->leak_list);
87 kmem_cache_free(extent_state_cache, state);
88 }
Eric Sandeen6d49ba12013-04-22 16:12:31 +000089}
David Sterba8d599ae2013-04-30 15:22:23 +000090
Josef Bacika5dee372013-12-13 10:02:44 -050091#define btrfs_debug_check_extent_io_range(tree, start, end) \
92 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000093static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050094 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000095{
Nikolay Borisov65a680f2018-11-01 14:09:49 +020096 struct inode *inode = tree->private_data;
97 u64 isize;
98
99 if (!inode || !is_data_inode(inode))
100 return;
101
102 isize = i_size_read(inode);
103 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
104 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
105 "%s: ino %llu isize %llu odd range [%llu,%llu]",
106 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
107 }
David Sterba8d599ae2013-04-30 15:22:23 +0000108}
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000109#else
110#define btrfs_leak_debug_add(new, head) do {} while (0)
111#define btrfs_leak_debug_del(entry) do {} while (0)
Josef Bacik33ca8322019-09-23 10:05:17 -0400112#define btrfs_extent_buffer_leak_debug_check() do {} while (0)
113#define btrfs_extent_state_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000114#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400115#endif
Chris Masond1310b22008-01-24 16:13:08 -0500116
Chris Masond1310b22008-01-24 16:13:08 -0500117struct tree_entry {
118 u64 start;
119 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500120 struct rb_node rb_node;
121};
122
123struct extent_page_data {
124 struct bio *bio;
Chris Mason771ed682008-11-06 22:02:51 -0500125 /* tells writepage not to lock the state bits for this range
126 * it still does the unlocking
127 */
Chris Masonffbd5172009-04-20 15:50:09 -0400128 unsigned int extent_locked:1;
129
Christoph Hellwig70fd7612016-11-01 07:40:10 -0600130 /* tells the submit_bio code to use REQ_SYNC */
Chris Masonffbd5172009-04-20 15:50:09 -0400131 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500132};
133
David Sterba57599c72018-03-01 17:56:34 +0100134static int add_extent_changeset(struct extent_state *state, unsigned bits,
Qu Wenruod38ed272015-10-12 14:53:37 +0800135 struct extent_changeset *changeset,
136 int set)
137{
138 int ret;
139
140 if (!changeset)
David Sterba57599c72018-03-01 17:56:34 +0100141 return 0;
Qu Wenruod38ed272015-10-12 14:53:37 +0800142 if (set && (state->state & bits) == bits)
David Sterba57599c72018-03-01 17:56:34 +0100143 return 0;
Qu Wenruofefdc552015-10-12 15:35:38 +0800144 if (!set && (state->state & bits) == 0)
David Sterba57599c72018-03-01 17:56:34 +0100145 return 0;
Qu Wenruod38ed272015-10-12 14:53:37 +0800146 changeset->bytes_changed += state->end - state->start + 1;
David Sterba53d32352017-02-13 13:42:29 +0100147 ret = ulist_add(&changeset->range_changed, state->start, state->end,
Qu Wenruod38ed272015-10-12 14:53:37 +0800148 GFP_ATOMIC);
David Sterba57599c72018-03-01 17:56:34 +0100149 return ret;
Qu Wenruod38ed272015-10-12 14:53:37 +0800150}
151
Qu Wenruobb58eb92019-01-25 13:09:15 +0800152static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
153 unsigned long bio_flags)
154{
155 blk_status_t ret = 0;
Qu Wenruobb58eb92019-01-25 13:09:15 +0800156 struct extent_io_tree *tree = bio->bi_private;
Qu Wenruobb58eb92019-01-25 13:09:15 +0800157
158 bio->bi_private = NULL;
159
160 if (tree->ops)
161 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
Nikolay Borisov50489a52019-04-10 19:46:04 +0300162 mirror_num, bio_flags);
Qu Wenruobb58eb92019-01-25 13:09:15 +0800163 else
164 btrfsic_submit_bio(bio);
165
166 return blk_status_to_errno(ret);
167}
168
Qu Wenruo30659762019-03-20 14:27:42 +0800169/* Cleanup unsubmitted bios */
170static void end_write_bio(struct extent_page_data *epd, int ret)
171{
172 if (epd->bio) {
173 epd->bio->bi_status = errno_to_blk_status(ret);
174 bio_endio(epd->bio);
175 epd->bio = NULL;
176 }
177}
178
Qu Wenruof4340622019-03-20 14:27:41 +0800179/*
180 * Submit bio from extent page data via submit_one_bio
181 *
182 * Return 0 if everything is OK.
183 * Return <0 for error.
184 */
185static int __must_check flush_write_bio(struct extent_page_data *epd)
Qu Wenruobb58eb92019-01-25 13:09:15 +0800186{
Qu Wenruof4340622019-03-20 14:27:41 +0800187 int ret = 0;
Qu Wenruobb58eb92019-01-25 13:09:15 +0800188
Qu Wenruof4340622019-03-20 14:27:41 +0800189 if (epd->bio) {
Qu Wenruobb58eb92019-01-25 13:09:15 +0800190 ret = submit_one_bio(epd->bio, 0, 0);
Qu Wenruof4340622019-03-20 14:27:41 +0800191 /*
192 * Clean up of epd->bio is handled by its endio function.
193 * And endio is either triggered by successful bio execution
194 * or the error handler of submit bio hook.
195 * So at this point, no matter what happened, we don't need
196 * to clean up epd->bio.
197 */
Qu Wenruobb58eb92019-01-25 13:09:15 +0800198 epd->bio = NULL;
199 }
Qu Wenruof4340622019-03-20 14:27:41 +0800200 return ret;
Qu Wenruobb58eb92019-01-25 13:09:15 +0800201}
David Sterbae2932ee2017-06-23 04:16:17 +0200202
Josef Bacik6f0d04f2019-09-23 10:05:18 -0400203int __init extent_state_cache_init(void)
Chris Masond1310b22008-01-24 16:13:08 -0500204{
David Sterba837e1972012-09-07 03:00:48 -0600205 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200206 sizeof(struct extent_state), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300207 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500208 if (!extent_state_cache)
209 return -ENOMEM;
Josef Bacik6f0d04f2019-09-23 10:05:18 -0400210 return 0;
211}
Chris Masond1310b22008-01-24 16:13:08 -0500212
Josef Bacik6f0d04f2019-09-23 10:05:18 -0400213int __init extent_io_init(void)
214{
David Sterba837e1972012-09-07 03:00:48 -0600215 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200216 sizeof(struct extent_buffer), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300217 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500218 if (!extent_buffer_cache)
Josef Bacik6f0d04f2019-09-23 10:05:18 -0400219 return -ENOMEM;
Chris Mason9be33952013-05-17 18:30:14 -0400220
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400221 if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
222 offsetof(struct btrfs_io_bio, bio),
223 BIOSET_NEED_BVECS))
Chris Mason9be33952013-05-17 18:30:14 -0400224 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700225
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400226 if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700227 goto free_bioset;
228
Chris Masond1310b22008-01-24 16:13:08 -0500229 return 0;
230
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700231free_bioset:
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400232 bioset_exit(&btrfs_bioset);
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700233
Chris Mason9be33952013-05-17 18:30:14 -0400234free_buffer_cache:
235 kmem_cache_destroy(extent_buffer_cache);
236 extent_buffer_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500237 return -ENOMEM;
238}
239
Josef Bacik6f0d04f2019-09-23 10:05:18 -0400240void __cold extent_state_cache_exit(void)
241{
242 btrfs_extent_state_leak_debug_check();
243 kmem_cache_destroy(extent_state_cache);
244}
245
David Sterbae67c7182018-02-19 17:24:18 +0100246void __cold extent_io_exit(void)
Chris Masond1310b22008-01-24 16:13:08 -0500247{
Josef Bacik33ca8322019-09-23 10:05:17 -0400248 btrfs_extent_buffer_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000249
250 /*
251 * Make sure all delayed rcu free are flushed before we
252 * destroy caches.
253 */
254 rcu_barrier();
Kinglong Mee5598e902016-01-29 21:36:35 +0800255 kmem_cache_destroy(extent_buffer_cache);
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -0400256 bioset_exit(&btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500257}
258
Josef Bacik41a2ee72020-01-17 09:02:21 -0500259/*
260 * For the file_extent_tree, we want to hold the inode lock when we lookup and
261 * update the disk_i_size, but lockdep will complain because our io_tree we hold
262 * the tree lock and get the inode lock when setting delalloc. These two things
263 * are unrelated, so make a class for the file_extent_tree so we don't get the
264 * two locking patterns mixed up.
265 */
266static struct lock_class_key file_extent_tree_class;
267
Qu Wenruoc258d6e2019-03-01 10:47:58 +0800268void extent_io_tree_init(struct btrfs_fs_info *fs_info,
Qu Wenruo43eb5f22019-03-01 10:47:59 +0800269 struct extent_io_tree *tree, unsigned int owner,
270 void *private_data)
Chris Masond1310b22008-01-24 16:13:08 -0500271{
Qu Wenruoc258d6e2019-03-01 10:47:58 +0800272 tree->fs_info = fs_info;
Eric Paris6bef4d32010-02-23 19:43:04 +0000273 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500274 tree->ops = NULL;
275 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500276 spin_lock_init(&tree->lock);
Josef Bacikc6100a42017-05-05 11:57:13 -0400277 tree->private_data = private_data;
Qu Wenruo43eb5f22019-03-01 10:47:59 +0800278 tree->owner = owner;
Josef Bacik41a2ee72020-01-17 09:02:21 -0500279 if (owner == IO_TREE_INODE_FILE_EXTENT)
280 lockdep_set_class(&tree->lock, &file_extent_tree_class);
Chris Masond1310b22008-01-24 16:13:08 -0500281}
Chris Masond1310b22008-01-24 16:13:08 -0500282
Nikolay Borisov41e7acd2019-03-25 14:31:24 +0200283void extent_io_tree_release(struct extent_io_tree *tree)
284{
285 spin_lock(&tree->lock);
286 /*
287 * Do a single barrier for the waitqueue_active check here, the state
288 * of the waitqueue should not change once extent_io_tree_release is
289 * called.
290 */
291 smp_mb();
292 while (!RB_EMPTY_ROOT(&tree->state)) {
293 struct rb_node *node;
294 struct extent_state *state;
295
296 node = rb_first(&tree->state);
297 state = rb_entry(node, struct extent_state, rb_node);
298 rb_erase(&state->rb_node, &tree->state);
299 RB_CLEAR_NODE(&state->rb_node);
300 /*
301 * btree io trees aren't supposed to have tasks waiting for
302 * changes in the flags of extent states ever.
303 */
304 ASSERT(!waitqueue_active(&state->wq));
305 free_extent_state(state);
306
307 cond_resched_lock(&tree->lock);
308 }
309 spin_unlock(&tree->lock);
310}
311
Christoph Hellwigb2950862008-12-02 09:54:17 -0500312static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500313{
314 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500315
Michal Hocko3ba7ab22017-01-09 15:39:02 +0100316 /*
317 * The given mask might be not appropriate for the slab allocator,
318 * drop the unsupported bits
319 */
320 mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
Chris Masond1310b22008-01-24 16:13:08 -0500321 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400322 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500323 return state;
324 state->state = 0;
David Sterba47dc1962016-02-11 13:24:13 +0100325 state->failrec = NULL;
Filipe Manana27a35072014-07-06 20:09:59 +0100326 RB_CLEAR_NODE(&state->rb_node);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000327 btrfs_leak_debug_add(&state->leak_list, &states);
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200328 refcount_set(&state->refs, 1);
Chris Masond1310b22008-01-24 16:13:08 -0500329 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100330 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500331 return state;
332}
Chris Masond1310b22008-01-24 16:13:08 -0500333
Chris Mason4845e442010-05-25 20:56:50 -0400334void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500335{
Chris Masond1310b22008-01-24 16:13:08 -0500336 if (!state)
337 return;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200338 if (refcount_dec_and_test(&state->refs)) {
Filipe Manana27a35072014-07-06 20:09:59 +0100339 WARN_ON(extent_state_in_tree(state));
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000340 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100341 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500342 kmem_cache_free(extent_state_cache, state);
343 }
344}
Chris Masond1310b22008-01-24 16:13:08 -0500345
Filipe Mananaf2071b22014-02-12 15:05:53 +0000346static struct rb_node *tree_insert(struct rb_root *root,
347 struct rb_node *search_start,
348 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000349 struct rb_node *node,
350 struct rb_node ***p_in,
351 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500352{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000353 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500354 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500355 struct tree_entry *entry;
356
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000357 if (p_in && parent_in) {
358 p = *p_in;
359 parent = *parent_in;
360 goto do_insert;
361 }
362
Filipe Mananaf2071b22014-02-12 15:05:53 +0000363 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500364 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500365 parent = *p;
366 entry = rb_entry(parent, struct tree_entry, rb_node);
367
368 if (offset < entry->start)
369 p = &(*p)->rb_left;
370 else if (offset > entry->end)
371 p = &(*p)->rb_right;
372 else
373 return parent;
374 }
375
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000376do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500377 rb_link_node(node, parent, p);
378 rb_insert_color(node, root);
379 return NULL;
380}
381
Nikolay Borisov8666e632019-06-05 14:50:04 +0300382/**
383 * __etree_search - searche @tree for an entry that contains @offset. Such
384 * entry would have entry->start <= offset && entry->end >= offset.
385 *
386 * @tree - the tree to search
387 * @offset - offset that should fall within an entry in @tree
388 * @next_ret - pointer to the first entry whose range ends after @offset
389 * @prev - pointer to the first entry whose range begins before @offset
390 * @p_ret - pointer where new node should be anchored (used when inserting an
391 * entry in the tree)
392 * @parent_ret - points to entry which would have been the parent of the entry,
393 * containing @offset
394 *
395 * This function returns a pointer to the entry that contains @offset byte
396 * address. If no such entry exists, then NULL is returned and the other
397 * pointer arguments to the function are filled, otherwise the found entry is
398 * returned and other pointers are left untouched.
399 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500400static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000401 struct rb_node **next_ret,
Nikolay Borisov352646c2019-01-30 16:51:00 +0200402 struct rb_node **prev_ret,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000403 struct rb_node ***p_ret,
404 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500405{
Chris Mason80ea96b2008-02-01 14:51:59 -0500406 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000407 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500408 struct rb_node *prev = NULL;
409 struct rb_node *orig_prev = NULL;
410 struct tree_entry *entry;
411 struct tree_entry *prev_entry = NULL;
412
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000413 while (*n) {
414 prev = *n;
415 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500416 prev_entry = entry;
417
418 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000419 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500420 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000421 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500422 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000423 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500424 }
425
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000426 if (p_ret)
427 *p_ret = n;
428 if (parent_ret)
429 *parent_ret = prev;
430
Nikolay Borisov352646c2019-01-30 16:51:00 +0200431 if (next_ret) {
Chris Masond1310b22008-01-24 16:13:08 -0500432 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500433 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500434 prev = rb_next(prev);
435 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
436 }
Nikolay Borisov352646c2019-01-30 16:51:00 +0200437 *next_ret = prev;
Chris Masond1310b22008-01-24 16:13:08 -0500438 prev = orig_prev;
439 }
440
Nikolay Borisov352646c2019-01-30 16:51:00 +0200441 if (prev_ret) {
Chris Masond1310b22008-01-24 16:13:08 -0500442 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500443 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500444 prev = rb_prev(prev);
445 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
446 }
Nikolay Borisov352646c2019-01-30 16:51:00 +0200447 *prev_ret = prev;
Chris Masond1310b22008-01-24 16:13:08 -0500448 }
449 return NULL;
450}
451
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000452static inline struct rb_node *
453tree_search_for_insert(struct extent_io_tree *tree,
454 u64 offset,
455 struct rb_node ***p_ret,
456 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500457{
Nikolay Borisov352646c2019-01-30 16:51:00 +0200458 struct rb_node *next= NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500459 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500460
Nikolay Borisov352646c2019-01-30 16:51:00 +0200461 ret = __etree_search(tree, offset, &next, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500462 if (!ret)
Nikolay Borisov352646c2019-01-30 16:51:00 +0200463 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500464 return ret;
465}
466
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000467static inline struct rb_node *tree_search(struct extent_io_tree *tree,
468 u64 offset)
469{
470 return tree_search_for_insert(tree, offset, NULL, NULL);
471}
472
Chris Masond1310b22008-01-24 16:13:08 -0500473/*
474 * utility function to look for merge candidates inside a given range.
475 * Any extents with matching state are merged together into a single
476 * extent in the tree. Extents with EXTENT_IO in their state field
477 * are not merged because the end_io handlers need to be able to do
478 * operations on them without sleeping (or doing allocations/splits).
479 *
480 * This should be called with the tree lock held.
481 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000482static void merge_state(struct extent_io_tree *tree,
483 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500484{
485 struct extent_state *other;
486 struct rb_node *other_node;
487
Nikolay Borisov88826792019-03-14 15:28:31 +0200488 if (state->state & (EXTENT_LOCKED | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000489 return;
Chris Masond1310b22008-01-24 16:13:08 -0500490
491 other_node = rb_prev(&state->rb_node);
492 if (other_node) {
493 other = rb_entry(other_node, struct extent_state, rb_node);
494 if (other->end == state->start - 1 &&
495 other->state == state->state) {
Nikolay Borisov5c848192018-11-01 14:09:52 +0200496 if (tree->private_data &&
497 is_data_inode(tree->private_data))
498 btrfs_merge_delalloc_extent(tree->private_data,
499 state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500500 state->start = other->start;
Chris Masond1310b22008-01-24 16:13:08 -0500501 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100502 RB_CLEAR_NODE(&other->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500503 free_extent_state(other);
504 }
505 }
506 other_node = rb_next(&state->rb_node);
507 if (other_node) {
508 other = rb_entry(other_node, struct extent_state, rb_node);
509 if (other->start == state->end + 1 &&
510 other->state == state->state) {
Nikolay Borisov5c848192018-11-01 14:09:52 +0200511 if (tree->private_data &&
512 is_data_inode(tree->private_data))
513 btrfs_merge_delalloc_extent(tree->private_data,
514 state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400515 state->end = other->end;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400516 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100517 RB_CLEAR_NODE(&other->rb_node);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400518 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500519 }
520 }
Chris Masond1310b22008-01-24 16:13:08 -0500521}
522
Xiao Guangrong3150b692011-07-14 03:19:08 +0000523static void set_state_bits(struct extent_io_tree *tree,
Qu Wenruod38ed272015-10-12 14:53:37 +0800524 struct extent_state *state, unsigned *bits,
525 struct extent_changeset *changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000526
Chris Masond1310b22008-01-24 16:13:08 -0500527/*
528 * insert an extent_state struct into the tree. 'bits' are set on the
529 * struct before it is inserted.
530 *
531 * This may return -EEXIST if the extent is already there, in which case the
532 * state struct is freed.
533 *
534 * The tree lock is not taken internally. This is a utility function and
535 * probably isn't what you want to call (see set/clear_extent_bit).
536 */
537static int insert_state(struct extent_io_tree *tree,
538 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000539 struct rb_node ***p,
540 struct rb_node **parent,
Qu Wenruod38ed272015-10-12 14:53:37 +0800541 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500542{
543 struct rb_node *node;
544
David Sterba27922372019-06-18 20:00:05 +0200545 if (end < start) {
546 btrfs_err(tree->fs_info,
547 "insert state: end < start %llu %llu", end, start);
548 WARN_ON(1);
549 }
Chris Masond1310b22008-01-24 16:13:08 -0500550 state->start = start;
551 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400552
Qu Wenruod38ed272015-10-12 14:53:37 +0800553 set_state_bits(tree, state, bits, changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000554
Filipe Mananaf2071b22014-02-12 15:05:53 +0000555 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500556 if (node) {
557 struct extent_state *found;
558 found = rb_entry(node, struct extent_state, rb_node);
David Sterba27922372019-06-18 20:00:05 +0200559 btrfs_err(tree->fs_info,
560 "found node %llu %llu on insert of %llu %llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200561 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500562 return -EEXIST;
563 }
564 merge_state(tree, state);
565 return 0;
566}
567
568/*
569 * split a given extent state struct in two, inserting the preallocated
570 * struct 'prealloc' as the newly created second half. 'split' indicates an
571 * offset inside 'orig' where it should be split.
572 *
573 * Before calling,
574 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
575 * are two extent state structs in the tree:
576 * prealloc: [orig->start, split - 1]
577 * orig: [ split, orig->end ]
578 *
579 * The tree locks are not taken by this function. They need to be held
580 * by the caller.
581 */
582static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
583 struct extent_state *prealloc, u64 split)
584{
585 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400586
Nikolay Borisovabbb55f2018-11-01 14:09:53 +0200587 if (tree->private_data && is_data_inode(tree->private_data))
588 btrfs_split_delalloc_extent(tree->private_data, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400589
Chris Masond1310b22008-01-24 16:13:08 -0500590 prealloc->start = orig->start;
591 prealloc->end = split - 1;
592 prealloc->state = orig->state;
593 orig->start = split;
594
Filipe Mananaf2071b22014-02-12 15:05:53 +0000595 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
596 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500597 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500598 free_extent_state(prealloc);
599 return -EEXIST;
600 }
601 return 0;
602}
603
Li Zefancdc6a392012-03-12 16:39:48 +0800604static struct extent_state *next_state(struct extent_state *state)
605{
606 struct rb_node *next = rb_next(&state->rb_node);
607 if (next)
608 return rb_entry(next, struct extent_state, rb_node);
609 else
610 return NULL;
611}
612
Chris Masond1310b22008-01-24 16:13:08 -0500613/*
614 * utility function to clear some bits in an extent state struct.
Andrea Gelmini52042d82018-11-28 12:05:13 +0100615 * it will optionally wake up anyone waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500616 *
617 * If no bits are set on the state struct after clearing things, the
618 * struct is freed and removed from the tree
619 */
Li Zefancdc6a392012-03-12 16:39:48 +0800620static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
621 struct extent_state *state,
Qu Wenruofefdc552015-10-12 15:35:38 +0800622 unsigned *bits, int wake,
623 struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500624{
Li Zefancdc6a392012-03-12 16:39:48 +0800625 struct extent_state *next;
David Sterba9ee49a042015-01-14 19:52:13 +0100626 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
David Sterba57599c72018-03-01 17:56:34 +0100627 int ret;
Chris Masond1310b22008-01-24 16:13:08 -0500628
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400629 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500630 u64 range = state->end - state->start + 1;
631 WARN_ON(range > tree->dirty_bytes);
632 tree->dirty_bytes -= range;
633 }
Nikolay Borisova36bb5f2018-11-01 14:09:51 +0200634
635 if (tree->private_data && is_data_inode(tree->private_data))
636 btrfs_clear_delalloc_extent(tree->private_data, state, bits);
637
David Sterba57599c72018-03-01 17:56:34 +0100638 ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
639 BUG_ON(ret < 0);
Josef Bacik32c00af2009-10-08 13:34:05 -0400640 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500641 if (wake)
642 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400643 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800644 next = next_state(state);
Filipe Manana27a35072014-07-06 20:09:59 +0100645 if (extent_state_in_tree(state)) {
Chris Masond1310b22008-01-24 16:13:08 -0500646 rb_erase(&state->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100647 RB_CLEAR_NODE(&state->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500648 free_extent_state(state);
649 } else {
650 WARN_ON(1);
651 }
652 } else {
653 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800654 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500655 }
Li Zefancdc6a392012-03-12 16:39:48 +0800656 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500657}
658
Xiao Guangrong82337672011-04-20 06:44:57 +0000659static struct extent_state *
660alloc_extent_state_atomic(struct extent_state *prealloc)
661{
662 if (!prealloc)
663 prealloc = alloc_extent_state(GFP_ATOMIC);
664
665 return prealloc;
666}
667
Eric Sandeen48a3b632013-04-25 20:41:01 +0000668static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400669{
David Sterba05912a32018-07-18 19:23:45 +0200670 struct inode *inode = tree->private_data;
671
672 btrfs_panic(btrfs_sb(inode->i_sb), err,
673 "locking error: extent tree was modified by another thread while locked");
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400674}
675
Chris Masond1310b22008-01-24 16:13:08 -0500676/*
677 * clear some bits on a range in the tree. This may require splitting
678 * or inserting elements in the tree, so the gfp mask is used to
679 * indicate which allocations or sleeping are allowed.
680 *
681 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
682 * the given range from the tree regardless of state (ie for truncate).
683 *
684 * the range [start, end] is inclusive.
685 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100686 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500687 */
David Sterba66b0c882017-10-31 16:30:47 +0100688int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
Qu Wenruofefdc552015-10-12 15:35:38 +0800689 unsigned bits, int wake, int delete,
690 struct extent_state **cached_state,
691 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500692{
693 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400694 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500695 struct extent_state *prealloc = NULL;
696 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400697 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500698 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000699 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500700
Josef Bacika5dee372013-12-13 10:02:44 -0500701 btrfs_debug_check_extent_io_range(tree, start, end);
Qu Wenruoa1d19842019-03-01 10:48:00 +0800702 trace_btrfs_clear_extent_bit(tree, start, end - start + 1, bits);
David Sterba8d599ae2013-04-30 15:22:23 +0000703
Josef Bacik7ee9e442013-06-21 16:37:03 -0400704 if (bits & EXTENT_DELALLOC)
705 bits |= EXTENT_NORESERVE;
706
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400707 if (delete)
708 bits |= ~EXTENT_CTLBITS;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400709
Nikolay Borisov88826792019-03-14 15:28:31 +0200710 if (bits & (EXTENT_LOCKED | EXTENT_BOUNDARY))
Josef Bacik2ac55d42010-02-03 19:33:23 +0000711 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500712again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800713 if (!prealloc && gfpflags_allow_blocking(mask)) {
Filipe Mananac7bc6312014-11-03 14:12:57 +0000714 /*
715 * Don't care for allocation failure here because we might end
716 * up not needing the pre-allocated extent state at all, which
717 * is the case if we only have in the tree extent states that
718 * cover our input range and don't cover too any other range.
719 * If we end up needing a new extent state we allocate it later.
720 */
Chris Masond1310b22008-01-24 16:13:08 -0500721 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500722 }
723
Chris Masoncad321a2008-12-17 14:51:42 -0500724 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400725 if (cached_state) {
726 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000727
728 if (clear) {
729 *cached_state = NULL;
730 cached_state = NULL;
731 }
732
Filipe Manana27a35072014-07-06 20:09:59 +0100733 if (cached && extent_state_in_tree(cached) &&
734 cached->start <= start && cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000735 if (clear)
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200736 refcount_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400737 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400738 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400739 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000740 if (clear)
741 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400742 }
Chris Masond1310b22008-01-24 16:13:08 -0500743 /*
744 * this search will find the extents that end after
745 * our range starts
746 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500747 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500748 if (!node)
749 goto out;
750 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400751hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500752 if (state->start > end)
753 goto out;
754 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400755 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500756
Liu Bo04493142012-02-16 18:34:37 +0800757 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800758 if (!(state->state & bits)) {
759 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800760 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800761 }
Liu Bo04493142012-02-16 18:34:37 +0800762
Chris Masond1310b22008-01-24 16:13:08 -0500763 /*
764 * | ---- desired range ---- |
765 * | state | or
766 * | ------------- state -------------- |
767 *
768 * We need to split the extent we found, and may flip
769 * bits on second half.
770 *
771 * If the extent we found extends past our range, we
772 * just split and search again. It'll get split again
773 * the next time though.
774 *
775 * If the extent we found is inside our range, we clear
776 * the desired bit on it.
777 */
778
779 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000780 prealloc = alloc_extent_state_atomic(prealloc);
781 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500782 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400783 if (err)
784 extent_io_tree_panic(tree, err);
785
Chris Masond1310b22008-01-24 16:13:08 -0500786 prealloc = NULL;
787 if (err)
788 goto out;
789 if (state->end <= end) {
Qu Wenruofefdc552015-10-12 15:35:38 +0800790 state = clear_state_bit(tree, state, &bits, wake,
791 changeset);
Liu Bod1ac6e42012-05-10 18:10:39 +0800792 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500793 }
794 goto search_again;
795 }
796 /*
797 * | ---- desired range ---- |
798 * | state |
799 * We need to split the extent, and clear the bit
800 * on the first half
801 */
802 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000803 prealloc = alloc_extent_state_atomic(prealloc);
804 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500805 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400806 if (err)
807 extent_io_tree_panic(tree, err);
808
Chris Masond1310b22008-01-24 16:13:08 -0500809 if (wake)
810 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400811
Qu Wenruofefdc552015-10-12 15:35:38 +0800812 clear_state_bit(tree, prealloc, &bits, wake, changeset);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400813
Chris Masond1310b22008-01-24 16:13:08 -0500814 prealloc = NULL;
815 goto out;
816 }
Chris Mason42daec22009-09-23 19:51:09 -0400817
Qu Wenruofefdc552015-10-12 15:35:38 +0800818 state = clear_state_bit(tree, state, &bits, wake, changeset);
Liu Bo04493142012-02-16 18:34:37 +0800819next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400820 if (last_end == (u64)-1)
821 goto out;
822 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800823 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800824 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500825
826search_again:
827 if (start > end)
828 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500829 spin_unlock(&tree->lock);
Mel Gormand0164ad2015-11-06 16:28:21 -0800830 if (gfpflags_allow_blocking(mask))
Chris Masond1310b22008-01-24 16:13:08 -0500831 cond_resched();
832 goto again;
David Sterba7ab5cb22016-04-27 01:02:15 +0200833
834out:
835 spin_unlock(&tree->lock);
836 if (prealloc)
837 free_extent_state(prealloc);
838
839 return 0;
840
Chris Masond1310b22008-01-24 16:13:08 -0500841}
Chris Masond1310b22008-01-24 16:13:08 -0500842
Jeff Mahoney143bede2012-03-01 14:56:26 +0100843static void wait_on_state(struct extent_io_tree *tree,
844 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500845 __releases(tree->lock)
846 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500847{
848 DEFINE_WAIT(wait);
849 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500850 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500851 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500852 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500853 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500854}
855
856/*
857 * waits for one or more bits to clear on a range in the state tree.
858 * The range [start, end] is inclusive.
859 * The tree lock is taken by this function
860 */
David Sterba41074882013-04-29 13:38:46 +0000861static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
862 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500863{
864 struct extent_state *state;
865 struct rb_node *node;
866
Josef Bacika5dee372013-12-13 10:02:44 -0500867 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000868
Chris Masoncad321a2008-12-17 14:51:42 -0500869 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500870again:
871 while (1) {
872 /*
873 * this search will find all the extents that end after
874 * our range starts
875 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500876 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100877process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500878 if (!node)
879 break;
880
881 state = rb_entry(node, struct extent_state, rb_node);
882
883 if (state->start > end)
884 goto out;
885
886 if (state->state & bits) {
887 start = state->start;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200888 refcount_inc(&state->refs);
Chris Masond1310b22008-01-24 16:13:08 -0500889 wait_on_state(tree, state);
890 free_extent_state(state);
891 goto again;
892 }
893 start = state->end + 1;
894
895 if (start > end)
896 break;
897
Filipe Mananac50d3e72014-03-31 14:53:25 +0100898 if (!cond_resched_lock(&tree->lock)) {
899 node = rb_next(node);
900 goto process_node;
901 }
Chris Masond1310b22008-01-24 16:13:08 -0500902 }
903out:
Chris Masoncad321a2008-12-17 14:51:42 -0500904 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500905}
Chris Masond1310b22008-01-24 16:13:08 -0500906
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000907static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500908 struct extent_state *state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800909 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500910{
David Sterba9ee49a042015-01-14 19:52:13 +0100911 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
David Sterba57599c72018-03-01 17:56:34 +0100912 int ret;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400913
Nikolay Borisove06a1fc2018-11-01 14:09:50 +0200914 if (tree->private_data && is_data_inode(tree->private_data))
915 btrfs_set_delalloc_extent(tree->private_data, state, bits);
916
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400917 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500918 u64 range = state->end - state->start + 1;
919 tree->dirty_bytes += range;
920 }
David Sterba57599c72018-03-01 17:56:34 +0100921 ret = add_extent_changeset(state, bits_to_set, changeset, 1);
922 BUG_ON(ret < 0);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400923 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500924}
925
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100926static void cache_state_if_flags(struct extent_state *state,
927 struct extent_state **cached_ptr,
David Sterba9ee49a042015-01-14 19:52:13 +0100928 unsigned flags)
Chris Mason2c64c532009-09-02 15:04:12 -0400929{
930 if (cached_ptr && !(*cached_ptr)) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100931 if (!flags || (state->state & flags)) {
Chris Mason2c64c532009-09-02 15:04:12 -0400932 *cached_ptr = state;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +0200933 refcount_inc(&state->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400934 }
935 }
936}
937
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100938static void cache_state(struct extent_state *state,
939 struct extent_state **cached_ptr)
940{
941 return cache_state_if_flags(state, cached_ptr,
Nikolay Borisov88826792019-03-14 15:28:31 +0200942 EXTENT_LOCKED | EXTENT_BOUNDARY);
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100943}
944
Chris Masond1310b22008-01-24 16:13:08 -0500945/*
Chris Mason1edbb732009-09-02 13:24:36 -0400946 * set some bits on a range in the tree. This may require allocations or
947 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500948 *
Chris Mason1edbb732009-09-02 13:24:36 -0400949 * If any of the exclusive bits are set, this will fail with -EEXIST if some
950 * part of the range already has the desired bits set. The start of the
951 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500952 *
Chris Mason1edbb732009-09-02 13:24:36 -0400953 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500954 */
Chris Mason1edbb732009-09-02 13:24:36 -0400955
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100956static int __must_check
957__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100958 unsigned bits, unsigned exclusive_bits,
David Sterba41074882013-04-29 13:38:46 +0000959 u64 *failed_start, struct extent_state **cached_state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800960 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500961{
962 struct extent_state *state;
963 struct extent_state *prealloc = NULL;
964 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000965 struct rb_node **p;
966 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500967 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500968 u64 last_start;
969 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400970
Josef Bacika5dee372013-12-13 10:02:44 -0500971 btrfs_debug_check_extent_io_range(tree, start, end);
Qu Wenruoa1d19842019-03-01 10:48:00 +0800972 trace_btrfs_set_extent_bit(tree, start, end - start + 1, bits);
David Sterba8d599ae2013-04-30 15:22:23 +0000973
Chris Masond1310b22008-01-24 16:13:08 -0500974again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800975 if (!prealloc && gfpflags_allow_blocking(mask)) {
David Sterba059f7912016-04-27 01:03:45 +0200976 /*
977 * Don't care for allocation failure here because we might end
978 * up not needing the pre-allocated extent state at all, which
979 * is the case if we only have in the tree extent states that
980 * cover our input range and don't cover too any other range.
981 * If we end up needing a new extent state we allocate it later.
982 */
Chris Masond1310b22008-01-24 16:13:08 -0500983 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500984 }
985
Chris Masoncad321a2008-12-17 14:51:42 -0500986 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400987 if (cached_state && *cached_state) {
988 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400989 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +0100990 extent_state_in_tree(state)) {
Chris Mason9655d292009-09-02 15:22:30 -0400991 node = &state->rb_node;
992 goto hit_next;
993 }
994 }
Chris Masond1310b22008-01-24 16:13:08 -0500995 /*
996 * this search will find all the extents that end after
997 * our range starts.
998 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000999 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -05001000 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +00001001 prealloc = alloc_extent_state_atomic(prealloc);
1002 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001003 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001004 &p, &parent, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001005 if (err)
1006 extent_io_tree_panic(tree, err);
1007
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001008 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001009 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05001010 goto out;
1011 }
Chris Masond1310b22008-01-24 16:13:08 -05001012 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -04001013hit_next:
Chris Masond1310b22008-01-24 16:13:08 -05001014 last_start = state->start;
1015 last_end = state->end;
1016
1017 /*
1018 * | ---- desired range ---- |
1019 * | state |
1020 *
1021 * Just lock what we found and keep going
1022 */
1023 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -04001024 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001025 *failed_start = state->start;
1026 err = -EEXIST;
1027 goto out;
1028 }
Chris Mason42daec22009-09-23 19:51:09 -04001029
Qu Wenruod38ed272015-10-12 14:53:37 +08001030 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -04001031 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001032 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -04001033 if (last_end == (u64)-1)
1034 goto out;
1035 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001036 state = next_state(state);
1037 if (start < end && state && state->start == start &&
1038 !need_resched())
1039 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -05001040 goto search_again;
1041 }
1042
1043 /*
1044 * | ---- desired range ---- |
1045 * | state |
1046 * or
1047 * | ------------- state -------------- |
1048 *
1049 * We need to split the extent we found, and may flip bits on
1050 * second half.
1051 *
1052 * If the extent we found extends past our
1053 * range, we just split and search again. It'll get split
1054 * again the next time though.
1055 *
1056 * If the extent we found is inside our range, we set the
1057 * desired bit on it.
1058 */
1059 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -04001060 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001061 *failed_start = start;
1062 err = -EEXIST;
1063 goto out;
1064 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001065
1066 prealloc = alloc_extent_state_atomic(prealloc);
1067 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001068 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001069 if (err)
1070 extent_io_tree_panic(tree, err);
1071
Chris Masond1310b22008-01-24 16:13:08 -05001072 prealloc = NULL;
1073 if (err)
1074 goto out;
1075 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001076 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -04001077 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001078 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -04001079 if (last_end == (u64)-1)
1080 goto out;
1081 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001082 state = next_state(state);
1083 if (start < end && state && state->start == start &&
1084 !need_resched())
1085 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -05001086 }
1087 goto search_again;
1088 }
1089 /*
1090 * | ---- desired range ---- |
1091 * | state | or | state |
1092 *
1093 * There's a hole, we need to insert something in it and
1094 * ignore the extent we found.
1095 */
1096 if (state->start > start) {
1097 u64 this_end;
1098 if (end < last_start)
1099 this_end = end;
1100 else
Chris Masond3977122009-01-05 21:25:51 -05001101 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +00001102
1103 prealloc = alloc_extent_state_atomic(prealloc);
1104 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +00001105
1106 /*
1107 * Avoid to free 'prealloc' if it can be merged with
1108 * the later extent.
1109 */
Chris Masond1310b22008-01-24 16:13:08 -05001110 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001111 NULL, NULL, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001112 if (err)
1113 extent_io_tree_panic(tree, err);
1114
Chris Mason2c64c532009-09-02 15:04:12 -04001115 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001116 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05001117 start = this_end + 1;
1118 goto search_again;
1119 }
1120 /*
1121 * | ---- desired range ---- |
1122 * | state |
1123 * We need to split the extent, and set the bit
1124 * on the first half
1125 */
1126 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -04001127 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001128 *failed_start = start;
1129 err = -EEXIST;
1130 goto out;
1131 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001132
1133 prealloc = alloc_extent_state_atomic(prealloc);
1134 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001135 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001136 if (err)
1137 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -05001138
Qu Wenruod38ed272015-10-12 14:53:37 +08001139 set_state_bits(tree, prealloc, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -04001140 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001141 merge_state(tree, prealloc);
1142 prealloc = NULL;
1143 goto out;
1144 }
1145
David Sterbab5a4ba142016-04-27 01:02:15 +02001146search_again:
1147 if (start > end)
1148 goto out;
1149 spin_unlock(&tree->lock);
1150 if (gfpflags_allow_blocking(mask))
1151 cond_resched();
1152 goto again;
Chris Masond1310b22008-01-24 16:13:08 -05001153
1154out:
Chris Masoncad321a2008-12-17 14:51:42 -05001155 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001156 if (prealloc)
1157 free_extent_state(prealloc);
1158
1159 return err;
1160
Chris Masond1310b22008-01-24 16:13:08 -05001161}
Chris Masond1310b22008-01-24 16:13:08 -05001162
David Sterba41074882013-04-29 13:38:46 +00001163int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001164 unsigned bits, u64 * failed_start,
David Sterba41074882013-04-29 13:38:46 +00001165 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001166{
1167 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001168 cached_state, mask, NULL);
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001169}
1170
1171
Josef Bacik462d6fa2011-09-26 13:56:12 -04001172/**
Liu Bo10983f22012-07-11 15:26:19 +08001173 * convert_extent_bit - convert all bits in a given range from one bit to
1174 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001175 * @tree: the io tree to search
1176 * @start: the start offset in bytes
1177 * @end: the end offset in bytes (inclusive)
1178 * @bits: the bits to set in this range
1179 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001180 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001181 *
1182 * This will go through and set bits for the given range. If any states exist
1183 * already in this range they are set with the given bit and cleared of the
1184 * clear_bits. This is only meant to be used by things that are mergeable, ie
1185 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1186 * boundary bits like LOCK.
David Sterba210aa272016-04-26 23:54:39 +02001187 *
1188 * All allocations are done with GFP_NOFS.
Josef Bacik462d6fa2011-09-26 13:56:12 -04001189 */
1190int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001191 unsigned bits, unsigned clear_bits,
David Sterba210aa272016-04-26 23:54:39 +02001192 struct extent_state **cached_state)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001193{
1194 struct extent_state *state;
1195 struct extent_state *prealloc = NULL;
1196 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001197 struct rb_node **p;
1198 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001199 int err = 0;
1200 u64 last_start;
1201 u64 last_end;
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001202 bool first_iteration = true;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001203
Josef Bacika5dee372013-12-13 10:02:44 -05001204 btrfs_debug_check_extent_io_range(tree, start, end);
Qu Wenruoa1d19842019-03-01 10:48:00 +08001205 trace_btrfs_convert_extent_bit(tree, start, end - start + 1, bits,
1206 clear_bits);
David Sterba8d599ae2013-04-30 15:22:23 +00001207
Josef Bacik462d6fa2011-09-26 13:56:12 -04001208again:
David Sterba210aa272016-04-26 23:54:39 +02001209 if (!prealloc) {
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001210 /*
1211 * Best effort, don't worry if extent state allocation fails
1212 * here for the first iteration. We might have a cached state
1213 * that matches exactly the target range, in which case no
1214 * extent state allocations are needed. We'll only know this
1215 * after locking the tree.
1216 */
David Sterba210aa272016-04-26 23:54:39 +02001217 prealloc = alloc_extent_state(GFP_NOFS);
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001218 if (!prealloc && !first_iteration)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001219 return -ENOMEM;
1220 }
1221
1222 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001223 if (cached_state && *cached_state) {
1224 state = *cached_state;
1225 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +01001226 extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001227 node = &state->rb_node;
1228 goto hit_next;
1229 }
1230 }
1231
Josef Bacik462d6fa2011-09-26 13:56:12 -04001232 /*
1233 * this search will find all the extents that end after
1234 * our range starts.
1235 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001236 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001237 if (!node) {
1238 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001239 if (!prealloc) {
1240 err = -ENOMEM;
1241 goto out;
1242 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001243 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001244 &p, &parent, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001245 if (err)
1246 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001247 cache_state(prealloc, cached_state);
1248 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001249 goto out;
1250 }
1251 state = rb_entry(node, struct extent_state, rb_node);
1252hit_next:
1253 last_start = state->start;
1254 last_end = state->end;
1255
1256 /*
1257 * | ---- desired range ---- |
1258 * | state |
1259 *
1260 * Just lock what we found and keep going
1261 */
1262 if (state->start == start && state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001263 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001264 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001265 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001266 if (last_end == (u64)-1)
1267 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001268 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001269 if (start < end && state && state->start == start &&
1270 !need_resched())
1271 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001272 goto search_again;
1273 }
1274
1275 /*
1276 * | ---- desired range ---- |
1277 * | state |
1278 * or
1279 * | ------------- state -------------- |
1280 *
1281 * We need to split the extent we found, and may flip bits on
1282 * second half.
1283 *
1284 * If the extent we found extends past our
1285 * range, we just split and search again. It'll get split
1286 * again the next time though.
1287 *
1288 * If the extent we found is inside our range, we set the
1289 * desired bit on it.
1290 */
1291 if (state->start < start) {
1292 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001293 if (!prealloc) {
1294 err = -ENOMEM;
1295 goto out;
1296 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001297 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001298 if (err)
1299 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001300 prealloc = NULL;
1301 if (err)
1302 goto out;
1303 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001304 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001305 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001306 state = clear_state_bit(tree, state, &clear_bits, 0,
1307 NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001308 if (last_end == (u64)-1)
1309 goto out;
1310 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001311 if (start < end && state && state->start == start &&
1312 !need_resched())
1313 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001314 }
1315 goto search_again;
1316 }
1317 /*
1318 * | ---- desired range ---- |
1319 * | state | or | state |
1320 *
1321 * There's a hole, we need to insert something in it and
1322 * ignore the extent we found.
1323 */
1324 if (state->start > start) {
1325 u64 this_end;
1326 if (end < last_start)
1327 this_end = end;
1328 else
1329 this_end = last_start - 1;
1330
1331 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001332 if (!prealloc) {
1333 err = -ENOMEM;
1334 goto out;
1335 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001336
1337 /*
1338 * Avoid to free 'prealloc' if it can be merged with
1339 * the later extent.
1340 */
1341 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001342 NULL, NULL, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001343 if (err)
1344 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001345 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001346 prealloc = NULL;
1347 start = this_end + 1;
1348 goto search_again;
1349 }
1350 /*
1351 * | ---- desired range ---- |
1352 * | state |
1353 * We need to split the extent, and set the bit
1354 * on the first half
1355 */
1356 if (state->start <= end && state->end > end) {
1357 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001358 if (!prealloc) {
1359 err = -ENOMEM;
1360 goto out;
1361 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001362
1363 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001364 if (err)
1365 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001366
Qu Wenruod38ed272015-10-12 14:53:37 +08001367 set_state_bits(tree, prealloc, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001368 cache_state(prealloc, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001369 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001370 prealloc = NULL;
1371 goto out;
1372 }
1373
Josef Bacik462d6fa2011-09-26 13:56:12 -04001374search_again:
1375 if (start > end)
1376 goto out;
1377 spin_unlock(&tree->lock);
David Sterba210aa272016-04-26 23:54:39 +02001378 cond_resched();
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001379 first_iteration = false;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001380 goto again;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001381
1382out:
1383 spin_unlock(&tree->lock);
1384 if (prealloc)
1385 free_extent_state(prealloc);
1386
1387 return err;
1388}
1389
Chris Masond1310b22008-01-24 16:13:08 -05001390/* wrappers around set/clear extent bit */
Qu Wenruod38ed272015-10-12 14:53:37 +08001391int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba2c53b912016-04-26 23:54:39 +02001392 unsigned bits, struct extent_changeset *changeset)
Qu Wenruod38ed272015-10-12 14:53:37 +08001393{
1394 /*
1395 * We don't support EXTENT_LOCKED yet, as current changeset will
1396 * record any bits changed, so for EXTENT_LOCKED case, it will
1397 * either fail with -EEXIST or changeset will record the whole
1398 * range.
1399 */
1400 BUG_ON(bits & EXTENT_LOCKED);
1401
David Sterba2c53b912016-04-26 23:54:39 +02001402 return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
Qu Wenruod38ed272015-10-12 14:53:37 +08001403 changeset);
1404}
1405
Nikolay Borisov4ca73652019-03-27 14:24:10 +02001406int set_extent_bits_nowait(struct extent_io_tree *tree, u64 start, u64 end,
1407 unsigned bits)
1408{
1409 return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL,
1410 GFP_NOWAIT, NULL);
1411}
1412
Qu Wenruofefdc552015-10-12 15:35:38 +08001413int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1414 unsigned bits, int wake, int delete,
David Sterbaae0f1622017-10-31 16:37:52 +01001415 struct extent_state **cached)
Qu Wenruofefdc552015-10-12 15:35:38 +08001416{
1417 return __clear_extent_bit(tree, start, end, bits, wake, delete,
David Sterbaae0f1622017-10-31 16:37:52 +01001418 cached, GFP_NOFS, NULL);
Qu Wenruofefdc552015-10-12 15:35:38 +08001419}
1420
Qu Wenruofefdc552015-10-12 15:35:38 +08001421int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaf734c442016-04-26 23:54:39 +02001422 unsigned bits, struct extent_changeset *changeset)
Qu Wenruofefdc552015-10-12 15:35:38 +08001423{
1424 /*
1425 * Don't support EXTENT_LOCKED case, same reason as
1426 * set_record_extent_bits().
1427 */
1428 BUG_ON(bits & EXTENT_LOCKED);
1429
David Sterbaf734c442016-04-26 23:54:39 +02001430 return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
Qu Wenruofefdc552015-10-12 15:35:38 +08001431 changeset);
1432}
1433
Chris Masond352ac62008-09-29 15:18:18 -04001434/*
1435 * either insert or lock state struct between start and end use mask to tell
1436 * us if waiting is desired.
1437 */
Chris Mason1edbb732009-09-02 13:24:36 -04001438int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaff13db42015-12-03 14:30:40 +01001439 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001440{
1441 int err;
1442 u64 failed_start;
David Sterba9ee49a042015-01-14 19:52:13 +01001443
Chris Masond1310b22008-01-24 16:13:08 -05001444 while (1) {
David Sterbaff13db42015-12-03 14:30:40 +01001445 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001446 EXTENT_LOCKED, &failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001447 cached_state, GFP_NOFS, NULL);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001448 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001449 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1450 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001451 } else
Chris Masond1310b22008-01-24 16:13:08 -05001452 break;
Chris Masond1310b22008-01-24 16:13:08 -05001453 WARN_ON(start > end);
1454 }
1455 return err;
1456}
Chris Masond1310b22008-01-24 16:13:08 -05001457
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001458int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001459{
1460 int err;
1461 u64 failed_start;
1462
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001463 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
Qu Wenruod38ed272015-10-12 14:53:37 +08001464 &failed_start, NULL, GFP_NOFS, NULL);
Yan Zheng66435582008-10-30 14:19:50 -04001465 if (err == -EEXIST) {
1466 if (failed_start > start)
1467 clear_extent_bit(tree, start, failed_start - 1,
David Sterbaae0f1622017-10-31 16:37:52 +01001468 EXTENT_LOCKED, 1, 0, NULL);
Josef Bacik25179202008-10-29 14:49:05 -04001469 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001470 }
Josef Bacik25179202008-10-29 14:49:05 -04001471 return 1;
1472}
Josef Bacik25179202008-10-29 14:49:05 -04001473
David Sterbabd1fa4f2015-12-03 13:08:59 +01001474void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001475{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001476 unsigned long index = start >> PAGE_SHIFT;
1477 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001478 struct page *page;
1479
1480 while (index <= end_index) {
1481 page = find_get_page(inode->i_mapping, index);
1482 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1483 clear_page_dirty_for_io(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001484 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001485 index++;
1486 }
Chris Mason4adaa612013-03-26 13:07:00 -04001487}
1488
David Sterbaf6311572015-12-03 13:08:59 +01001489void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001490{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001491 unsigned long index = start >> PAGE_SHIFT;
1492 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001493 struct page *page;
1494
1495 while (index <= end_index) {
1496 page = find_get_page(inode->i_mapping, index);
1497 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Mason4adaa612013-03-26 13:07:00 -04001498 __set_page_dirty_nobuffers(page);
Konstantin Khebnikov8d386332015-02-11 15:26:55 -08001499 account_page_redirty(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001500 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001501 index++;
1502 }
Chris Mason4adaa612013-03-26 13:07:00 -04001503}
1504
Chris Masond352ac62008-09-29 15:18:18 -04001505/* find the first state struct with 'bits' set after 'start', and
1506 * return it. tree->lock must be held. NULL will returned if
1507 * nothing was found after 'start'
1508 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001509static struct extent_state *
1510find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +01001511 u64 start, unsigned bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001512{
1513 struct rb_node *node;
1514 struct extent_state *state;
1515
1516 /*
1517 * this search will find all the extents that end after
1518 * our range starts.
1519 */
1520 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001521 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001522 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001523
Chris Masond3977122009-01-05 21:25:51 -05001524 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001525 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001526 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001527 return state;
Chris Masond3977122009-01-05 21:25:51 -05001528
Chris Masond7fc6402008-02-18 12:12:38 -05001529 node = rb_next(node);
1530 if (!node)
1531 break;
1532 }
1533out:
1534 return NULL;
1535}
Chris Masond7fc6402008-02-18 12:12:38 -05001536
Chris Masond352ac62008-09-29 15:18:18 -04001537/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001538 * find the first offset in the io tree with 'bits' set. zero is
1539 * returned if we find something, and *start_ret and *end_ret are
1540 * set to reflect the state struct that was found.
1541 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001542 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001543 */
1544int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba9ee49a042015-01-14 19:52:13 +01001545 u64 *start_ret, u64 *end_ret, unsigned bits,
Josef Bacike6138872012-09-27 17:07:30 -04001546 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001547{
1548 struct extent_state *state;
1549 int ret = 1;
1550
1551 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001552 if (cached_state && *cached_state) {
1553 state = *cached_state;
Filipe Manana27a35072014-07-06 20:09:59 +01001554 if (state->end == start - 1 && extent_state_in_tree(state)) {
Liu Bo9688e9a2018-08-23 03:14:53 +08001555 while ((state = next_state(state)) != NULL) {
Josef Bacike6138872012-09-27 17:07:30 -04001556 if (state->state & bits)
1557 goto got_it;
Josef Bacike6138872012-09-27 17:07:30 -04001558 }
1559 free_extent_state(*cached_state);
1560 *cached_state = NULL;
1561 goto out;
1562 }
1563 free_extent_state(*cached_state);
1564 *cached_state = NULL;
1565 }
1566
Xiao Guangrong69261c42011-07-14 03:19:45 +00001567 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001568got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001569 if (state) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +01001570 cache_state_if_flags(state, cached_state, 0);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001571 *start_ret = state->start;
1572 *end_ret = state->end;
1573 ret = 0;
1574 }
Josef Bacike6138872012-09-27 17:07:30 -04001575out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001576 spin_unlock(&tree->lock);
1577 return ret;
1578}
1579
Nikolay Borisov45bfcfc2019-03-27 14:24:17 +02001580/**
Josef Bacik41a2ee72020-01-17 09:02:21 -05001581 * find_contiguous_extent_bit: find a contiguous area of bits
1582 * @tree - io tree to check
1583 * @start - offset to start the search from
1584 * @start_ret - the first offset we found with the bits set
1585 * @end_ret - the final contiguous range of the bits that were set
1586 * @bits - bits to look for
1587 *
1588 * set_extent_bit and clear_extent_bit can temporarily split contiguous ranges
1589 * to set bits appropriately, and then merge them again. During this time it
1590 * will drop the tree->lock, so use this helper if you want to find the actual
1591 * contiguous area for given bits. We will search to the first bit we find, and
1592 * then walk down the tree until we find a non-contiguous area. The area
1593 * returned will be the full contiguous area with the bits set.
1594 */
1595int find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start,
1596 u64 *start_ret, u64 *end_ret, unsigned bits)
1597{
1598 struct extent_state *state;
1599 int ret = 1;
1600
1601 spin_lock(&tree->lock);
1602 state = find_first_extent_bit_state(tree, start, bits);
1603 if (state) {
1604 *start_ret = state->start;
1605 *end_ret = state->end;
1606 while ((state = next_state(state)) != NULL) {
1607 if (state->start > (*end_ret + 1))
1608 break;
1609 *end_ret = state->end;
1610 }
1611 ret = 0;
1612 }
1613 spin_unlock(&tree->lock);
1614 return ret;
1615}
1616
1617/**
Nikolay Borisov1eaebb32019-06-03 13:06:02 +03001618 * find_first_clear_extent_bit - find the first range that has @bits not set.
1619 * This range could start before @start.
Nikolay Borisov45bfcfc2019-03-27 14:24:17 +02001620 *
1621 * @tree - the tree to search
1622 * @start - the offset at/after which the found extent should start
1623 * @start_ret - records the beginning of the range
1624 * @end_ret - records the end of the range (inclusive)
1625 * @bits - the set of bits which must be unset
1626 *
1627 * Since unallocated range is also considered one which doesn't have the bits
1628 * set it's possible that @end_ret contains -1, this happens in case the range
1629 * spans (last_range_end, end of device]. In this case it's up to the caller to
1630 * trim @end_ret to the appropriate size.
1631 */
1632void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start,
1633 u64 *start_ret, u64 *end_ret, unsigned bits)
1634{
1635 struct extent_state *state;
1636 struct rb_node *node, *prev = NULL, *next;
1637
1638 spin_lock(&tree->lock);
1639
1640 /* Find first extent with bits cleared */
1641 while (1) {
1642 node = __etree_search(tree, start, &next, &prev, NULL, NULL);
Nikolay Borisov5750c372020-01-27 11:59:26 +02001643 if (!node && !next && !prev) {
1644 /*
1645 * Tree is completely empty, send full range and let
1646 * caller deal with it
1647 */
1648 *start_ret = 0;
1649 *end_ret = -1;
1650 goto out;
1651 } else if (!node && !next) {
1652 /*
1653 * We are past the last allocated chunk, set start at
1654 * the end of the last extent.
1655 */
1656 state = rb_entry(prev, struct extent_state, rb_node);
1657 *start_ret = state->end + 1;
1658 *end_ret = -1;
1659 goto out;
1660 } else if (!node) {
Nikolay Borisov45bfcfc2019-03-27 14:24:17 +02001661 node = next;
Nikolay Borisov45bfcfc2019-03-27 14:24:17 +02001662 }
Nikolay Borisov1eaebb32019-06-03 13:06:02 +03001663 /*
1664 * At this point 'node' either contains 'start' or start is
1665 * before 'node'
1666 */
Nikolay Borisov45bfcfc2019-03-27 14:24:17 +02001667 state = rb_entry(node, struct extent_state, rb_node);
Nikolay Borisov1eaebb32019-06-03 13:06:02 +03001668
1669 if (in_range(start, state->start, state->end - state->start + 1)) {
1670 if (state->state & bits) {
1671 /*
1672 * |--range with bits sets--|
1673 * |
1674 * start
1675 */
1676 start = state->end + 1;
1677 } else {
1678 /*
1679 * 'start' falls within a range that doesn't
1680 * have the bits set, so take its start as
1681 * the beginning of the desired range
1682 *
1683 * |--range with bits cleared----|
1684 * |
1685 * start
1686 */
1687 *start_ret = state->start;
1688 break;
1689 }
Nikolay Borisov45bfcfc2019-03-27 14:24:17 +02001690 } else {
Nikolay Borisov1eaebb32019-06-03 13:06:02 +03001691 /*
1692 * |---prev range---|---hole/unset---|---node range---|
1693 * |
1694 * start
1695 *
1696 * or
1697 *
1698 * |---hole/unset--||--first node--|
1699 * 0 |
1700 * start
1701 */
1702 if (prev) {
1703 state = rb_entry(prev, struct extent_state,
1704 rb_node);
1705 *start_ret = state->end + 1;
1706 } else {
1707 *start_ret = 0;
1708 }
Nikolay Borisov45bfcfc2019-03-27 14:24:17 +02001709 break;
1710 }
1711 }
1712
1713 /*
1714 * Find the longest stretch from start until an entry which has the
1715 * bits set
1716 */
1717 while (1) {
1718 state = rb_entry(node, struct extent_state, rb_node);
1719 if (state->end >= start && !(state->state & bits)) {
1720 *end_ret = state->end;
1721 } else {
1722 *end_ret = state->start - 1;
1723 break;
1724 }
1725
1726 node = rb_next(node);
1727 if (!node)
1728 break;
1729 }
1730out:
1731 spin_unlock(&tree->lock);
1732}
1733
Xiao Guangrong69261c42011-07-14 03:19:45 +00001734/*
Chris Masond352ac62008-09-29 15:18:18 -04001735 * find a contiguous range of bytes in the file marked as delalloc, not
1736 * more than 'max_bytes'. start and end are used to return the range,
1737 *
Lu Fengqi3522e902018-11-29 11:33:38 +08001738 * true is returned if we find something, false if nothing was in the tree
Chris Masond352ac62008-09-29 15:18:18 -04001739 */
Josef Bacik083e75e2019-09-23 10:05:20 -04001740bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start,
1741 u64 *end, u64 max_bytes,
1742 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001743{
1744 struct rb_node *node;
1745 struct extent_state *state;
1746 u64 cur_start = *start;
Lu Fengqi3522e902018-11-29 11:33:38 +08001747 bool found = false;
Chris Masond1310b22008-01-24 16:13:08 -05001748 u64 total_bytes = 0;
1749
Chris Masoncad321a2008-12-17 14:51:42 -05001750 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001751
Chris Masond1310b22008-01-24 16:13:08 -05001752 /*
1753 * this search will find all the extents that end after
1754 * our range starts.
1755 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001756 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001757 if (!node) {
Lu Fengqi3522e902018-11-29 11:33:38 +08001758 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001759 goto out;
1760 }
1761
Chris Masond3977122009-01-05 21:25:51 -05001762 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001763 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001764 if (found && (state->start != cur_start ||
1765 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001766 goto out;
1767 }
1768 if (!(state->state & EXTENT_DELALLOC)) {
1769 if (!found)
1770 *end = state->end;
1771 goto out;
1772 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001773 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001774 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001775 *cached_state = state;
Elena Reshetovab7ac31b2017-03-03 10:55:19 +02001776 refcount_inc(&state->refs);
Josef Bacikc2a128d2010-02-02 21:19:11 +00001777 }
Lu Fengqi3522e902018-11-29 11:33:38 +08001778 found = true;
Chris Masond1310b22008-01-24 16:13:08 -05001779 *end = state->end;
1780 cur_start = state->end + 1;
1781 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001782 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001783 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001784 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001785 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001786 break;
1787 }
1788out:
Chris Masoncad321a2008-12-17 14:51:42 -05001789 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001790 return found;
1791}
1792
Liu Boda2c7002017-02-10 16:41:05 +01001793static int __process_pages_contig(struct address_space *mapping,
1794 struct page *locked_page,
1795 pgoff_t start_index, pgoff_t end_index,
1796 unsigned long page_ops, pgoff_t *index_ret);
1797
Jeff Mahoney143bede2012-03-01 14:56:26 +01001798static noinline void __unlock_for_delalloc(struct inode *inode,
1799 struct page *locked_page,
1800 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001801{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001802 unsigned long index = start >> PAGE_SHIFT;
1803 unsigned long end_index = end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001804
Liu Bo76c00212017-02-10 16:42:14 +01001805 ASSERT(locked_page);
Chris Masonc8b97812008-10-29 14:49:59 -04001806 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001807 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001808
Liu Bo76c00212017-02-10 16:42:14 +01001809 __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1810 PAGE_UNLOCK, NULL);
Chris Masonc8b97812008-10-29 14:49:59 -04001811}
1812
1813static noinline int lock_delalloc_pages(struct inode *inode,
1814 struct page *locked_page,
1815 u64 delalloc_start,
1816 u64 delalloc_end)
1817{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001818 unsigned long index = delalloc_start >> PAGE_SHIFT;
Liu Bo76c00212017-02-10 16:42:14 +01001819 unsigned long index_ret = index;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001820 unsigned long end_index = delalloc_end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001821 int ret;
Chris Masonc8b97812008-10-29 14:49:59 -04001822
Liu Bo76c00212017-02-10 16:42:14 +01001823 ASSERT(locked_page);
Chris Masonc8b97812008-10-29 14:49:59 -04001824 if (index == locked_page->index && index == end_index)
1825 return 0;
1826
Liu Bo76c00212017-02-10 16:42:14 +01001827 ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1828 end_index, PAGE_LOCK, &index_ret);
1829 if (ret == -EAGAIN)
1830 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1831 (u64)index_ret << PAGE_SHIFT);
Chris Masonc8b97812008-10-29 14:49:59 -04001832 return ret;
1833}
1834
1835/*
Lu Fengqi3522e902018-11-29 11:33:38 +08001836 * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1837 * more than @max_bytes. @Start and @end are used to return the range,
Chris Masonc8b97812008-10-29 14:49:59 -04001838 *
Lu Fengqi3522e902018-11-29 11:33:38 +08001839 * Return: true if we find something
1840 * false if nothing was in the tree
Chris Masonc8b97812008-10-29 14:49:59 -04001841 */
Johannes Thumshirnce9f9672018-11-19 10:38:17 +01001842EXPORT_FOR_TESTS
Lu Fengqi3522e902018-11-29 11:33:38 +08001843noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
Josef Bacik294e30f2013-10-09 12:00:56 -04001844 struct page *locked_page, u64 *start,
Nikolay Borisov917aace2018-10-26 14:43:20 +03001845 u64 *end)
Chris Masonc8b97812008-10-29 14:49:59 -04001846{
Goldwyn Rodrigues99780592019-06-21 10:02:54 -05001847 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Nikolay Borisov917aace2018-10-26 14:43:20 +03001848 u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001849 u64 delalloc_start;
1850 u64 delalloc_end;
Lu Fengqi3522e902018-11-29 11:33:38 +08001851 bool found;
Chris Mason9655d292009-09-02 15:22:30 -04001852 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001853 int ret;
1854 int loops = 0;
1855
1856again:
1857 /* step one, find a bunch of delalloc bytes starting at start */
1858 delalloc_start = *start;
1859 delalloc_end = 0;
Josef Bacik083e75e2019-09-23 10:05:20 -04001860 found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1861 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001862 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001863 *start = delalloc_start;
1864 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001865 free_extent_state(cached_state);
Lu Fengqi3522e902018-11-29 11:33:38 +08001866 return false;
Chris Masonc8b97812008-10-29 14:49:59 -04001867 }
1868
1869 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001870 * start comes from the offset of locked_page. We have to lock
1871 * pages in order, so we can't process delalloc bytes before
1872 * locked_page
1873 */
Chris Masond3977122009-01-05 21:25:51 -05001874 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001875 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001876
1877 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001878 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001879 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001880 if (delalloc_end + 1 - delalloc_start > max_bytes)
1881 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001882
Chris Masonc8b97812008-10-29 14:49:59 -04001883 /* step two, lock all the pages after the page that has start */
1884 ret = lock_delalloc_pages(inode, locked_page,
1885 delalloc_start, delalloc_end);
Nikolay Borisov9bfd61d2018-10-26 14:43:21 +03001886 ASSERT(!ret || ret == -EAGAIN);
Chris Masonc8b97812008-10-29 14:49:59 -04001887 if (ret == -EAGAIN) {
1888 /* some of the pages are gone, lets avoid looping by
1889 * shortening the size of the delalloc range we're searching
1890 */
Chris Mason9655d292009-09-02 15:22:30 -04001891 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001892 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001893 if (!loops) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001894 max_bytes = PAGE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001895 loops = 1;
1896 goto again;
1897 } else {
Lu Fengqi3522e902018-11-29 11:33:38 +08001898 found = false;
Chris Masonc8b97812008-10-29 14:49:59 -04001899 goto out_failed;
1900 }
1901 }
Chris Masonc8b97812008-10-29 14:49:59 -04001902
1903 /* step three, lock the state bits for the whole range */
David Sterbaff13db42015-12-03 14:30:40 +01001904 lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001905
1906 /* then test to make sure it is all still delalloc */
1907 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001908 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001909 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001910 unlock_extent_cached(tree, delalloc_start, delalloc_end,
David Sterbae43bbe52017-12-12 21:43:52 +01001911 &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001912 __unlock_for_delalloc(inode, locked_page,
1913 delalloc_start, delalloc_end);
1914 cond_resched();
1915 goto again;
1916 }
Chris Mason9655d292009-09-02 15:22:30 -04001917 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001918 *start = delalloc_start;
1919 *end = delalloc_end;
1920out_failed:
1921 return found;
1922}
1923
Liu Boda2c7002017-02-10 16:41:05 +01001924static int __process_pages_contig(struct address_space *mapping,
1925 struct page *locked_page,
1926 pgoff_t start_index, pgoff_t end_index,
1927 unsigned long page_ops, pgoff_t *index_ret)
Chris Masonc8b97812008-10-29 14:49:59 -04001928{
Liu Bo873695b2017-02-02 17:49:22 -08001929 unsigned long nr_pages = end_index - start_index + 1;
Liu Boda2c7002017-02-10 16:41:05 +01001930 unsigned long pages_locked = 0;
Liu Bo873695b2017-02-02 17:49:22 -08001931 pgoff_t index = start_index;
Chris Masonc8b97812008-10-29 14:49:59 -04001932 struct page *pages[16];
Liu Bo873695b2017-02-02 17:49:22 -08001933 unsigned ret;
Liu Boda2c7002017-02-10 16:41:05 +01001934 int err = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001935 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001936
Liu Boda2c7002017-02-10 16:41:05 +01001937 if (page_ops & PAGE_LOCK) {
1938 ASSERT(page_ops == PAGE_LOCK);
1939 ASSERT(index_ret && *index_ret == start_index);
1940 }
1941
Filipe Manana704de492014-10-06 22:14:22 +01001942 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
Liu Bo873695b2017-02-02 17:49:22 -08001943 mapping_set_error(mapping, -EIO);
Filipe Manana704de492014-10-06 22:14:22 +01001944
Chris Masond3977122009-01-05 21:25:51 -05001945 while (nr_pages > 0) {
Liu Bo873695b2017-02-02 17:49:22 -08001946 ret = find_get_pages_contig(mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001947 min_t(unsigned long,
1948 nr_pages, ARRAY_SIZE(pages)), pages);
Liu Boda2c7002017-02-10 16:41:05 +01001949 if (ret == 0) {
1950 /*
1951 * Only if we're going to lock these pages,
1952 * can we find nothing at @index.
1953 */
1954 ASSERT(page_ops & PAGE_LOCK);
Liu Bo49d4a332017-03-06 18:20:56 -08001955 err = -EAGAIN;
1956 goto out;
Liu Boda2c7002017-02-10 16:41:05 +01001957 }
Chris Mason8b62b722009-09-02 16:53:46 -04001958
Liu Boda2c7002017-02-10 16:41:05 +01001959 for (i = 0; i < ret; i++) {
Josef Bacikc2790a22013-07-29 11:20:47 -04001960 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001961 SetPagePrivate2(pages[i]);
1962
Chris Mason1d53c9e2019-07-10 12:28:16 -07001963 if (locked_page && pages[i] == locked_page) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001964 put_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001965 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001966 continue;
1967 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001968 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001969 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001970 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001971 set_page_writeback(pages[i]);
Filipe Manana704de492014-10-06 22:14:22 +01001972 if (page_ops & PAGE_SET_ERROR)
1973 SetPageError(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001974 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001975 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001976 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001977 unlock_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001978 if (page_ops & PAGE_LOCK) {
1979 lock_page(pages[i]);
1980 if (!PageDirty(pages[i]) ||
1981 pages[i]->mapping != mapping) {
1982 unlock_page(pages[i]);
1983 put_page(pages[i]);
1984 err = -EAGAIN;
1985 goto out;
1986 }
1987 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001988 put_page(pages[i]);
Liu Boda2c7002017-02-10 16:41:05 +01001989 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001990 }
1991 nr_pages -= ret;
1992 index += ret;
1993 cond_resched();
1994 }
Liu Boda2c7002017-02-10 16:41:05 +01001995out:
1996 if (err && index_ret)
1997 *index_ret = start_index + pages_locked - 1;
1998 return err;
Chris Masonc8b97812008-10-29 14:49:59 -04001999}
Chris Masonc8b97812008-10-29 14:49:59 -04002000
Liu Bo873695b2017-02-02 17:49:22 -08002001void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
Nikolay Borisov74e91942019-07-17 16:18:16 +03002002 struct page *locked_page,
2003 unsigned clear_bits,
2004 unsigned long page_ops)
Liu Bo873695b2017-02-02 17:49:22 -08002005{
2006 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
David Sterbaae0f1622017-10-31 16:37:52 +01002007 NULL);
Liu Bo873695b2017-02-02 17:49:22 -08002008
2009 __process_pages_contig(inode->i_mapping, locked_page,
2010 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
Liu Boda2c7002017-02-10 16:41:05 +01002011 page_ops, NULL);
Liu Bo873695b2017-02-02 17:49:22 -08002012}
2013
Chris Masond352ac62008-09-29 15:18:18 -04002014/*
2015 * count the number of bytes in the tree that have a given bit(s)
2016 * set. This can be fairly slow, except for EXTENT_DIRTY which is
2017 * cached. The total number found is returned.
2018 */
Chris Masond1310b22008-01-24 16:13:08 -05002019u64 count_range_bits(struct extent_io_tree *tree,
2020 u64 *start, u64 search_end, u64 max_bytes,
David Sterba9ee49a042015-01-14 19:52:13 +01002021 unsigned bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05002022{
2023 struct rb_node *node;
2024 struct extent_state *state;
2025 u64 cur_start = *start;
2026 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05002027 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002028 int found = 0;
2029
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05302030 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05002031 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05002032
Chris Masoncad321a2008-12-17 14:51:42 -05002033 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05002034 if (cur_start == 0 && bits == EXTENT_DIRTY) {
2035 total_bytes = tree->dirty_bytes;
2036 goto out;
2037 }
2038 /*
2039 * this search will find all the extents that end after
2040 * our range starts.
2041 */
Chris Mason80ea96b2008-02-01 14:51:59 -05002042 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05002043 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05002044 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05002045
Chris Masond3977122009-01-05 21:25:51 -05002046 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05002047 state = rb_entry(node, struct extent_state, rb_node);
2048 if (state->start > search_end)
2049 break;
Chris Masonec29ed52011-02-23 16:23:20 -05002050 if (contig && found && state->start > last + 1)
2051 break;
2052 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05002053 total_bytes += min(search_end, state->end) + 1 -
2054 max(cur_start, state->start);
2055 if (total_bytes >= max_bytes)
2056 break;
2057 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04002058 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05002059 found = 1;
2060 }
Chris Masonec29ed52011-02-23 16:23:20 -05002061 last = state->end;
2062 } else if (contig && found) {
2063 break;
Chris Masond1310b22008-01-24 16:13:08 -05002064 }
2065 node = rb_next(node);
2066 if (!node)
2067 break;
2068 }
2069out:
Chris Masoncad321a2008-12-17 14:51:42 -05002070 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05002071 return total_bytes;
2072}
Christoph Hellwigb2950862008-12-02 09:54:17 -05002073
Chris Masond352ac62008-09-29 15:18:18 -04002074/*
2075 * set the private field for a given byte offset in the tree. If there isn't
2076 * an extent_state there already, this does nothing.
2077 */
Josef Bacikb3f167a2019-09-23 10:05:21 -04002078int set_state_failrec(struct extent_io_tree *tree, u64 start,
2079 struct io_failure_record *failrec)
Chris Masond1310b22008-01-24 16:13:08 -05002080{
2081 struct rb_node *node;
2082 struct extent_state *state;
2083 int ret = 0;
2084
Chris Masoncad321a2008-12-17 14:51:42 -05002085 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05002086 /*
2087 * this search will find all the extents that end after
2088 * our range starts.
2089 */
Chris Mason80ea96b2008-02-01 14:51:59 -05002090 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04002091 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05002092 ret = -ENOENT;
2093 goto out;
2094 }
2095 state = rb_entry(node, struct extent_state, rb_node);
2096 if (state->start != start) {
2097 ret = -ENOENT;
2098 goto out;
2099 }
David Sterba47dc1962016-02-11 13:24:13 +01002100 state->failrec = failrec;
Chris Masond1310b22008-01-24 16:13:08 -05002101out:
Chris Masoncad321a2008-12-17 14:51:42 -05002102 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05002103 return ret;
2104}
2105
Josef Bacikb3f167a2019-09-23 10:05:21 -04002106int get_state_failrec(struct extent_io_tree *tree, u64 start,
2107 struct io_failure_record **failrec)
Chris Masond1310b22008-01-24 16:13:08 -05002108{
2109 struct rb_node *node;
2110 struct extent_state *state;
2111 int ret = 0;
2112
Chris Masoncad321a2008-12-17 14:51:42 -05002113 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05002114 /*
2115 * this search will find all the extents that end after
2116 * our range starts.
2117 */
Chris Mason80ea96b2008-02-01 14:51:59 -05002118 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04002119 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05002120 ret = -ENOENT;
2121 goto out;
2122 }
2123 state = rb_entry(node, struct extent_state, rb_node);
2124 if (state->start != start) {
2125 ret = -ENOENT;
2126 goto out;
2127 }
David Sterba47dc1962016-02-11 13:24:13 +01002128 *failrec = state->failrec;
Chris Masond1310b22008-01-24 16:13:08 -05002129out:
Chris Masoncad321a2008-12-17 14:51:42 -05002130 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05002131 return ret;
2132}
2133
2134/*
2135 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05002136 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05002137 * has the bits set. Otherwise, 1 is returned if any bit in the
2138 * range is found set.
2139 */
2140int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01002141 unsigned bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05002142{
2143 struct extent_state *state = NULL;
2144 struct rb_node *node;
2145 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002146
Chris Masoncad321a2008-12-17 14:51:42 -05002147 spin_lock(&tree->lock);
Filipe Manana27a35072014-07-06 20:09:59 +01002148 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
Josef Bacikdf98b6e2011-06-20 14:53:48 -04002149 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04002150 node = &cached->rb_node;
2151 else
2152 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05002153 while (node && start <= end) {
2154 state = rb_entry(node, struct extent_state, rb_node);
2155
2156 if (filled && state->start > start) {
2157 bitset = 0;
2158 break;
2159 }
2160
2161 if (state->start > end)
2162 break;
2163
2164 if (state->state & bits) {
2165 bitset = 1;
2166 if (!filled)
2167 break;
2168 } else if (filled) {
2169 bitset = 0;
2170 break;
2171 }
Chris Mason46562ce2009-09-23 20:23:16 -04002172
2173 if (state->end == (u64)-1)
2174 break;
2175
Chris Masond1310b22008-01-24 16:13:08 -05002176 start = state->end + 1;
2177 if (start > end)
2178 break;
2179 node = rb_next(node);
2180 if (!node) {
2181 if (filled)
2182 bitset = 0;
2183 break;
2184 }
2185 }
Chris Masoncad321a2008-12-17 14:51:42 -05002186 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05002187 return bitset;
2188}
Chris Masond1310b22008-01-24 16:13:08 -05002189
2190/*
2191 * helper function to set a given page up to date if all the
2192 * extents in the tree for that page are up to date
2193 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01002194static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05002195{
Miao Xie4eee4fa2012-12-21 09:17:45 +00002196 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002197 u64 end = start + PAGE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04002198 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05002199 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002200}
2201
Josef Bacik7870d082017-05-05 11:57:15 -04002202int free_io_failure(struct extent_io_tree *failure_tree,
2203 struct extent_io_tree *io_tree,
2204 struct io_failure_record *rec)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002205{
2206 int ret;
2207 int err = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002208
David Sterba47dc1962016-02-11 13:24:13 +01002209 set_state_failrec(failure_tree, rec->start, NULL);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002210 ret = clear_extent_bits(failure_tree, rec->start,
2211 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02002212 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002213 if (ret)
2214 err = ret;
2215
Josef Bacik7870d082017-05-05 11:57:15 -04002216 ret = clear_extent_bits(io_tree, rec->start,
David Woodhouse53b381b2013-01-29 18:40:14 -05002217 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02002218 EXTENT_DAMAGED);
David Woodhouse53b381b2013-01-29 18:40:14 -05002219 if (ret && !err)
2220 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002221
2222 kfree(rec);
2223 return err;
2224}
2225
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002226/*
2227 * this bypasses the standard btrfs submit functions deliberately, as
2228 * the standard behavior is to write all copies in a raid setup. here we only
2229 * want to write the one bad copy. so we do the mapping for ourselves and issue
2230 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002231 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002232 * actually prevents the read that triggered the error from finishing.
2233 * currently, there can be no more than two copies of every data bit. thus,
2234 * exactly one rewrite is required.
2235 */
Josef Bacik6ec656b2017-05-05 11:57:14 -04002236int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
2237 u64 length, u64 logical, struct page *page,
2238 unsigned int pg_offset, int mirror_num)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002239{
2240 struct bio *bio;
2241 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002242 u64 map_length = 0;
2243 u64 sector;
2244 struct btrfs_bio *bbio = NULL;
2245 int ret;
2246
Linus Torvalds1751e8a2017-11-27 13:05:09 -08002247 ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002248 BUG_ON(!mirror_num);
2249
David Sterbac5e4c3d2017-06-12 17:29:41 +02002250 bio = btrfs_io_bio_alloc(1);
Kent Overstreet4f024f32013-10-11 15:44:27 -07002251 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002252 map_length = length;
2253
Filipe Mananab5de8d02016-05-27 22:21:27 +01002254 /*
2255 * Avoid races with device replace and make sure our bbio has devices
2256 * associated to its stripes that don't go away while we are doing the
2257 * read repair operation.
2258 */
2259 btrfs_bio_counter_inc_blocked(fs_info);
Nikolay Borisove4ff5fb2017-07-19 10:48:42 +03002260 if (btrfs_is_parity_mirror(fs_info, logical, length)) {
Liu Boc7253282017-03-29 10:53:58 -07002261 /*
2262 * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2263 * to update all raid stripes, but here we just want to correct
2264 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2265 * stripe's dev and sector.
2266 */
2267 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
2268 &map_length, &bbio, 0);
2269 if (ret) {
2270 btrfs_bio_counter_dec(fs_info);
2271 bio_put(bio);
2272 return -EIO;
2273 }
2274 ASSERT(bbio->mirror_num == 1);
2275 } else {
2276 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
2277 &map_length, &bbio, mirror_num);
2278 if (ret) {
2279 btrfs_bio_counter_dec(fs_info);
2280 bio_put(bio);
2281 return -EIO;
2282 }
2283 BUG_ON(mirror_num != bbio->mirror_num);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002284 }
Liu Boc7253282017-03-29 10:53:58 -07002285
2286 sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002287 bio->bi_iter.bi_sector = sector;
Liu Boc7253282017-03-29 10:53:58 -07002288 dev = bbio->stripes[bbio->mirror_num - 1].dev;
Zhao Lei6e9606d2015-01-20 15:11:34 +08002289 btrfs_put_bbio(bbio);
Anand Jainebbede42017-12-04 12:54:52 +08002290 if (!dev || !dev->bdev ||
2291 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
Filipe Mananab5de8d02016-05-27 22:21:27 +01002292 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002293 bio_put(bio);
2294 return -EIO;
2295 }
Christoph Hellwig74d46992017-08-23 19:10:32 +02002296 bio_set_dev(bio, dev->bdev);
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002297 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
Miao Xieffdd2012014-09-12 18:44:00 +08002298 bio_add_page(bio, page, length, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002299
Mike Christie4e49ea42016-06-05 14:31:41 -05002300 if (btrfsic_submit_bio_wait(bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002301 /* try to remap that extent elsewhere? */
Filipe Mananab5de8d02016-05-27 22:21:27 +01002302 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002303 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002304 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002305 return -EIO;
2306 }
2307
David Sterbab14af3b2015-10-08 10:43:10 +02002308 btrfs_info_rl_in_rcu(fs_info,
2309 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
Josef Bacik6ec656b2017-05-05 11:57:14 -04002310 ino, start,
Miao Xie1203b682014-09-12 18:44:01 +08002311 rcu_str_deref(dev->name), sector);
Filipe Mananab5de8d02016-05-27 22:21:27 +01002312 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002313 bio_put(bio);
2314 return 0;
2315}
2316
David Sterba20a1fbf92019-03-20 11:23:44 +01002317int btrfs_repair_eb_io_failure(struct extent_buffer *eb, int mirror_num)
Josef Bacikea466792012-03-26 21:57:36 -04002318{
David Sterba20a1fbf92019-03-20 11:23:44 +01002319 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacikea466792012-03-26 21:57:36 -04002320 u64 start = eb->start;
David Sterbacc5e31a2018-03-01 18:20:27 +01002321 int i, num_pages = num_extent_pages(eb);
Chris Masond95603b2012-04-12 15:55:15 -04002322 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002323
David Howellsbc98a422017-07-17 08:45:34 +01002324 if (sb_rdonly(fs_info->sb))
Ilya Dryomov908960c2013-11-03 19:06:39 +02002325 return -EROFS;
2326
Josef Bacikea466792012-03-26 21:57:36 -04002327 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02002328 struct page *p = eb->pages[i];
Miao Xie1203b682014-09-12 18:44:01 +08002329
Josef Bacik6ec656b2017-05-05 11:57:14 -04002330 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
Miao Xie1203b682014-09-12 18:44:01 +08002331 start - page_offset(p), mirror_num);
Josef Bacikea466792012-03-26 21:57:36 -04002332 if (ret)
2333 break;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002334 start += PAGE_SIZE;
Josef Bacikea466792012-03-26 21:57:36 -04002335 }
2336
2337 return ret;
2338}
2339
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002340/*
2341 * each time an IO finishes, we do a fast check in the IO failure tree
2342 * to see if we need to process or clean up an io_failure_record
2343 */
Josef Bacik7870d082017-05-05 11:57:15 -04002344int clean_io_failure(struct btrfs_fs_info *fs_info,
2345 struct extent_io_tree *failure_tree,
2346 struct extent_io_tree *io_tree, u64 start,
2347 struct page *page, u64 ino, unsigned int pg_offset)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002348{
2349 u64 private;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002350 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002351 struct extent_state *state;
2352 int num_copies;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002353 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002354
2355 private = 0;
Josef Bacik7870d082017-05-05 11:57:15 -04002356 ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2357 EXTENT_DIRTY, 0);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002358 if (!ret)
2359 return 0;
2360
Josef Bacik7870d082017-05-05 11:57:15 -04002361 ret = get_state_failrec(failure_tree, start, &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002362 if (ret)
2363 return 0;
2364
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002365 BUG_ON(!failrec->this_mirror);
2366
2367 if (failrec->in_validation) {
2368 /* there was no real error, just free the record */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002369 btrfs_debug(fs_info,
2370 "clean_io_failure: freeing dummy error at %llu",
2371 failrec->start);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002372 goto out;
2373 }
David Howellsbc98a422017-07-17 08:45:34 +01002374 if (sb_rdonly(fs_info->sb))
Ilya Dryomov908960c2013-11-03 19:06:39 +02002375 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002376
Josef Bacik7870d082017-05-05 11:57:15 -04002377 spin_lock(&io_tree->lock);
2378 state = find_first_extent_bit_state(io_tree,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002379 failrec->start,
2380 EXTENT_LOCKED);
Josef Bacik7870d082017-05-05 11:57:15 -04002381 spin_unlock(&io_tree->lock);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002382
Miao Xie883d0de2013-07-25 19:22:35 +08002383 if (state && state->start <= failrec->start &&
2384 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002385 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2386 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002387 if (num_copies > 1) {
Josef Bacik7870d082017-05-05 11:57:15 -04002388 repair_io_failure(fs_info, ino, start, failrec->len,
2389 failrec->logical, page, pg_offset,
2390 failrec->failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002391 }
2392 }
2393
2394out:
Josef Bacik7870d082017-05-05 11:57:15 -04002395 free_io_failure(failure_tree, io_tree, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002396
Miao Xie454ff3d2014-09-12 18:43:58 +08002397 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002398}
2399
Miao Xief6124962014-09-12 18:44:04 +08002400/*
2401 * Can be called when
2402 * - hold extent lock
2403 * - under ordered extent
2404 * - the inode is freeing
2405 */
Nikolay Borisov7ab79562017-02-20 13:50:57 +02002406void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
Miao Xief6124962014-09-12 18:44:04 +08002407{
Nikolay Borisov7ab79562017-02-20 13:50:57 +02002408 struct extent_io_tree *failure_tree = &inode->io_failure_tree;
Miao Xief6124962014-09-12 18:44:04 +08002409 struct io_failure_record *failrec;
2410 struct extent_state *state, *next;
2411
2412 if (RB_EMPTY_ROOT(&failure_tree->state))
2413 return;
2414
2415 spin_lock(&failure_tree->lock);
2416 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2417 while (state) {
2418 if (state->start > end)
2419 break;
2420
2421 ASSERT(state->end <= end);
2422
2423 next = next_state(state);
2424
David Sterba47dc1962016-02-11 13:24:13 +01002425 failrec = state->failrec;
Miao Xief6124962014-09-12 18:44:04 +08002426 free_extent_state(state);
2427 kfree(failrec);
2428
2429 state = next;
2430 }
2431 spin_unlock(&failure_tree->lock);
2432}
2433
Miao Xie2fe63032014-09-12 18:43:59 +08002434int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
David Sterba47dc1962016-02-11 13:24:13 +01002435 struct io_failure_record **failrec_ret)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002436{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002437 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002438 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002439 struct extent_map *em;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002440 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2441 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2442 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002443 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002444 u64 logical;
2445
David Sterba47dc1962016-02-11 13:24:13 +01002446 ret = get_state_failrec(failure_tree, start, &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002447 if (ret) {
2448 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2449 if (!failrec)
2450 return -ENOMEM;
Miao Xie2fe63032014-09-12 18:43:59 +08002451
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002452 failrec->start = start;
2453 failrec->len = end - start + 1;
2454 failrec->this_mirror = 0;
2455 failrec->bio_flags = 0;
2456 failrec->in_validation = 0;
2457
2458 read_lock(&em_tree->lock);
2459 em = lookup_extent_mapping(em_tree, start, failrec->len);
2460 if (!em) {
2461 read_unlock(&em_tree->lock);
2462 kfree(failrec);
2463 return -EIO;
2464 }
2465
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002466 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002467 free_extent_map(em);
2468 em = NULL;
2469 }
2470 read_unlock(&em_tree->lock);
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002471 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002472 kfree(failrec);
2473 return -EIO;
2474 }
Miao Xie2fe63032014-09-12 18:43:59 +08002475
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002476 logical = start - em->start;
2477 logical = em->block_start + logical;
2478 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2479 logical = em->block_start;
2480 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2481 extent_set_compress_type(&failrec->bio_flags,
2482 em->compress_type);
2483 }
Miao Xie2fe63032014-09-12 18:43:59 +08002484
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002485 btrfs_debug(fs_info,
2486 "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2487 logical, start, failrec->len);
Miao Xie2fe63032014-09-12 18:43:59 +08002488
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002489 failrec->logical = logical;
2490 free_extent_map(em);
2491
2492 /* set the bits in the private failure tree */
2493 ret = set_extent_bits(failure_tree, start, end,
David Sterbaceeb0ae2016-04-26 23:54:39 +02002494 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002495 if (ret >= 0)
David Sterba47dc1962016-02-11 13:24:13 +01002496 ret = set_state_failrec(failure_tree, start, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002497 /* set the bits in the inode's tree */
2498 if (ret >= 0)
David Sterbaceeb0ae2016-04-26 23:54:39 +02002499 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002500 if (ret < 0) {
2501 kfree(failrec);
2502 return ret;
2503 }
2504 } else {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002505 btrfs_debug(fs_info,
2506 "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2507 failrec->logical, failrec->start, failrec->len,
2508 failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002509 /*
2510 * when data can be on disk more than twice, add to failrec here
2511 * (e.g. with a list for failed_mirror) to make
2512 * clean_io_failure() clean all those errors at once.
2513 */
2514 }
Miao Xie2fe63032014-09-12 18:43:59 +08002515
2516 *failrec_ret = failrec;
2517
2518 return 0;
2519}
2520
Ming Leia0b60d72017-12-18 20:22:11 +08002521bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
Miao Xie2fe63032014-09-12 18:43:59 +08002522 struct io_failure_record *failrec, int failed_mirror)
2523{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002524 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002525 int num_copies;
2526
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002527 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002528 if (num_copies == 1) {
2529 /*
2530 * we only have a single copy of the data, so don't bother with
2531 * all the retry and error correction code that follows. no
2532 * matter what the error is, it is very likely to persist.
2533 */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002534 btrfs_debug(fs_info,
2535 "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2536 num_copies, failrec->this_mirror, failed_mirror);
Liu Boc3cfb652017-07-13 15:00:50 -07002537 return false;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002538 }
2539
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002540 /*
2541 * there are two premises:
2542 * a) deliver good data to the caller
2543 * b) correct the bad sectors on disk
2544 */
Ming Leia0b60d72017-12-18 20:22:11 +08002545 if (failed_bio_pages > 1) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002546 /*
2547 * to fulfill b), we need to know the exact failing sectors, as
2548 * we don't want to rewrite any more than the failed ones. thus,
2549 * we need separate read requests for the failed bio
2550 *
2551 * if the following BUG_ON triggers, our validation request got
2552 * merged. we need separate requests for our algorithm to work.
2553 */
2554 BUG_ON(failrec->in_validation);
2555 failrec->in_validation = 1;
2556 failrec->this_mirror = failed_mirror;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002557 } else {
2558 /*
2559 * we're ready to fulfill a) and b) alongside. get a good copy
2560 * of the failed sector and if we succeed, we have setup
2561 * everything for repair_io_failure to do the rest for us.
2562 */
2563 if (failrec->in_validation) {
2564 BUG_ON(failrec->this_mirror != failed_mirror);
2565 failrec->in_validation = 0;
2566 failrec->this_mirror = 0;
2567 }
2568 failrec->failed_mirror = failed_mirror;
2569 failrec->this_mirror++;
2570 if (failrec->this_mirror == failed_mirror)
2571 failrec->this_mirror++;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002572 }
2573
Miao Xiefacc8a222013-07-25 19:22:34 +08002574 if (failrec->this_mirror > num_copies) {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002575 btrfs_debug(fs_info,
2576 "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2577 num_copies, failrec->this_mirror, failed_mirror);
Liu Boc3cfb652017-07-13 15:00:50 -07002578 return false;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002579 }
2580
Liu Boc3cfb652017-07-13 15:00:50 -07002581 return true;
Miao Xie2fe63032014-09-12 18:43:59 +08002582}
2583
2584
2585struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2586 struct io_failure_record *failrec,
2587 struct page *page, int pg_offset, int icsum,
Miao Xie8b110e32014-09-12 18:44:03 +08002588 bio_end_io_t *endio_func, void *data)
Miao Xie2fe63032014-09-12 18:43:59 +08002589{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002590 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002591 struct bio *bio;
2592 struct btrfs_io_bio *btrfs_failed_bio;
2593 struct btrfs_io_bio *btrfs_bio;
2594
David Sterbac5e4c3d2017-06-12 17:29:41 +02002595 bio = btrfs_io_bio_alloc(1);
Miao Xie2fe63032014-09-12 18:43:59 +08002596 bio->bi_end_io = endio_func;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002597 bio->bi_iter.bi_sector = failrec->logical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002598 bio->bi_iter.bi_size = 0;
Miao Xie8b110e32014-09-12 18:44:03 +08002599 bio->bi_private = data;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002600
Miao Xiefacc8a222013-07-25 19:22:34 +08002601 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2602 if (btrfs_failed_bio->csum) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002603 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2604
2605 btrfs_bio = btrfs_io_bio(bio);
2606 btrfs_bio->csum = btrfs_bio->csum_inline;
Miao Xie2fe63032014-09-12 18:43:59 +08002607 icsum *= csum_size;
2608 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
Miao Xiefacc8a222013-07-25 19:22:34 +08002609 csum_size);
2610 }
2611
Miao Xie2fe63032014-09-12 18:43:59 +08002612 bio_add_page(bio, page, failrec->len, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002613
Miao Xie2fe63032014-09-12 18:43:59 +08002614 return bio;
2615}
2616
2617/*
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002618 * This is a generic handler for readpage errors. If other copies exist, read
2619 * those and write back good data to the failed position. Does not investigate
2620 * in remapping the failed extent elsewhere, hoping the device will be smart
2621 * enough to do this as needed
Miao Xie2fe63032014-09-12 18:43:59 +08002622 */
Miao Xie2fe63032014-09-12 18:43:59 +08002623static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2624 struct page *page, u64 start, u64 end,
2625 int failed_mirror)
2626{
2627 struct io_failure_record *failrec;
2628 struct inode *inode = page->mapping->host;
2629 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Josef Bacik7870d082017-05-05 11:57:15 -04002630 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
Miao Xie2fe63032014-09-12 18:43:59 +08002631 struct bio *bio;
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002632 int read_mode = 0;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002633 blk_status_t status;
Miao Xie2fe63032014-09-12 18:43:59 +08002634 int ret;
Christoph Hellwig8a2ee442019-02-15 19:13:07 +08002635 unsigned failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT;
Miao Xie2fe63032014-09-12 18:43:59 +08002636
Mike Christie1f7ad752016-06-05 14:31:51 -05002637 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
Miao Xie2fe63032014-09-12 18:43:59 +08002638
2639 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2640 if (ret)
2641 return ret;
2642
Ming Leia0b60d72017-12-18 20:22:11 +08002643 if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
Liu Boc3cfb652017-07-13 15:00:50 -07002644 failed_mirror)) {
Josef Bacik7870d082017-05-05 11:57:15 -04002645 free_io_failure(failure_tree, tree, failrec);
Miao Xie2fe63032014-09-12 18:43:59 +08002646 return -EIO;
2647 }
2648
Ming Leia0b60d72017-12-18 20:22:11 +08002649 if (failed_bio_pages > 1)
Christoph Hellwig70fd7612016-11-01 07:40:10 -06002650 read_mode |= REQ_FAILFAST_DEV;
Miao Xie2fe63032014-09-12 18:43:59 +08002651
2652 phy_offset >>= inode->i_sb->s_blocksize_bits;
2653 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2654 start - page_offset(page),
Miao Xie8b110e32014-09-12 18:44:03 +08002655 (int)phy_offset, failed_bio->bi_end_io,
2656 NULL);
David Sterbaebcc3262018-06-29 10:56:53 +02002657 bio->bi_opf = REQ_OP_READ | read_mode;
Miao Xie2fe63032014-09-12 18:43:59 +08002658
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002659 btrfs_debug(btrfs_sb(inode->i_sb),
2660 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2661 read_mode, failrec->this_mirror, failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002662
Linus Torvalds8c27cb32017-07-05 16:41:23 -07002663 status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
Nikolay Borisov50489a52019-04-10 19:46:04 +03002664 failrec->bio_flags);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002665 if (status) {
Josef Bacik7870d082017-05-05 11:57:15 -04002666 free_io_failure(failure_tree, tree, failrec);
Miao Xie6c387ab2014-09-12 18:43:57 +08002667 bio_put(bio);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002668 ret = blk_status_to_errno(status);
Miao Xie6c387ab2014-09-12 18:43:57 +08002669 }
2670
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002671 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002672}
2673
Chris Masond1310b22008-01-24 16:13:08 -05002674/* lots and lots of room for performance fixes in the end_bio funcs */
2675
David Sterbab5227c02015-12-03 13:08:59 +01002676void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
Jeff Mahoney87826df2012-02-15 16:23:57 +01002677{
2678 int uptodate = (err == 0);
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002679 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002680
Nikolay Borisovc6297322018-11-08 10:18:08 +02002681 btrfs_writepage_endio_finish_ordered(page, start, end, uptodate);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002682
Jeff Mahoney87826df2012-02-15 16:23:57 +01002683 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002684 ClearPageUptodate(page);
2685 SetPageError(page);
Colin Ian Kingbff5baf2017-05-09 18:14:01 +01002686 ret = err < 0 ? err : -EIO;
Liu Bo5dca6ee2014-05-12 12:47:36 +08002687 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002688 }
Jeff Mahoney87826df2012-02-15 16:23:57 +01002689}
2690
Chris Masond1310b22008-01-24 16:13:08 -05002691/*
2692 * after a writepage IO is done, we need to:
2693 * clear the uptodate bits on error
2694 * clear the writeback bits in the extent tree for this IO
2695 * end_page_writeback if the page has no more pending IO
2696 *
2697 * Scheduling is not allowed, so the extent state tree is expected
2698 * to have one and only one object corresponding to this IO.
2699 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002700static void end_bio_extent_writepage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002701{
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002702 int error = blk_status_to_errno(bio->bi_status);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002703 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002704 u64 start;
2705 u64 end;
Ming Lei6dc4f102019-02-15 19:13:19 +08002706 struct bvec_iter_all iter_all;
Chris Masond1310b22008-01-24 16:13:08 -05002707
David Sterbac09abff2017-07-13 18:10:07 +02002708 ASSERT(!bio_flagged(bio, BIO_CLONED));
Christoph Hellwig2b070cf2019-04-25 09:03:00 +02002709 bio_for_each_segment_all(bvec, bio, iter_all) {
Chris Masond1310b22008-01-24 16:13:08 -05002710 struct page *page = bvec->bv_page;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002711 struct inode *inode = page->mapping->host;
2712 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
David Woodhouse902b22f2008-08-20 08:51:49 -04002713
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002714 /* We always issue full-page reads, but if some block
2715 * in a page fails to read, blk_update_request() will
2716 * advance bv_offset and adjust bv_len to compensate.
2717 * Print a warning for nonzero offsets, and an error
2718 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002719 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2720 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002721 btrfs_err(fs_info,
Frank Holtonefe120a2013-12-20 11:37:06 -05002722 "partial page write in btrfs with offset %u and length %u",
2723 bvec->bv_offset, bvec->bv_len);
2724 else
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002725 btrfs_info(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04002726 "incomplete page write in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002727 bvec->bv_offset, bvec->bv_len);
2728 }
Chris Masond1310b22008-01-24 16:13:08 -05002729
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002730 start = page_offset(page);
2731 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002732
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002733 end_extent_writepage(page, error, start, end);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002734 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002735 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002736
Chris Masond1310b22008-01-24 16:13:08 -05002737 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002738}
2739
Miao Xie883d0de2013-07-25 19:22:35 +08002740static void
2741endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2742 int uptodate)
2743{
2744 struct extent_state *cached = NULL;
2745 u64 end = start + len - 1;
2746
2747 if (uptodate && tree->track_uptodate)
2748 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
David Sterbad810a4b2017-12-07 18:52:54 +01002749 unlock_extent_cached_atomic(tree, start, end, &cached);
Miao Xie883d0de2013-07-25 19:22:35 +08002750}
2751
Chris Masond1310b22008-01-24 16:13:08 -05002752/*
2753 * after a readpage IO is done, we need to:
2754 * clear the uptodate bits on error
2755 * set the uptodate bits if things worked
2756 * set the page up to date if all extents in the tree are uptodate
2757 * clear the lock bit in the extent tree
2758 * unlock the page if there are no other extents locked for it
2759 *
2760 * Scheduling is not allowed, so the extent state tree is expected
2761 * to have one and only one object corresponding to this IO.
2762 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002763static void end_bio_extent_readpage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002764{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002765 struct bio_vec *bvec;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002766 int uptodate = !bio->bi_status;
Miao Xiefacc8a222013-07-25 19:22:34 +08002767 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
Josef Bacik7870d082017-05-05 11:57:15 -04002768 struct extent_io_tree *tree, *failure_tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002769 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002770 u64 start;
2771 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002772 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002773 u64 extent_start = 0;
2774 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002775 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002776 int ret;
Ming Lei6dc4f102019-02-15 19:13:19 +08002777 struct bvec_iter_all iter_all;
Chris Masond1310b22008-01-24 16:13:08 -05002778
David Sterbac09abff2017-07-13 18:10:07 +02002779 ASSERT(!bio_flagged(bio, BIO_CLONED));
Christoph Hellwig2b070cf2019-04-25 09:03:00 +02002780 bio_for_each_segment_all(bvec, bio, iter_all) {
Chris Masond1310b22008-01-24 16:13:08 -05002781 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002782 struct inode *inode = page->mapping->host;
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002783 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002784 bool data_inode = btrfs_ino(BTRFS_I(inode))
2785 != BTRFS_BTREE_INODE_OBJECTID;
Arne Jansen507903b2011-04-06 10:02:20 +00002786
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002787 btrfs_debug(fs_info,
2788 "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02002789 (u64)bio->bi_iter.bi_sector, bio->bi_status,
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002790 io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002791 tree = &BTRFS_I(inode)->io_tree;
Josef Bacik7870d082017-05-05 11:57:15 -04002792 failure_tree = &BTRFS_I(inode)->io_failure_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002793
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002794 /* We always issue full-page reads, but if some block
2795 * in a page fails to read, blk_update_request() will
2796 * advance bv_offset and adjust bv_len to compensate.
2797 * Print a warning for nonzero offsets, and an error
2798 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002799 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2800 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002801 btrfs_err(fs_info,
2802 "partial page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002803 bvec->bv_offset, bvec->bv_len);
2804 else
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002805 btrfs_info(fs_info,
2806 "incomplete page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002807 bvec->bv_offset, bvec->bv_len);
2808 }
Chris Masond1310b22008-01-24 16:13:08 -05002809
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002810 start = page_offset(page);
2811 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002812 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002813
Chris Mason9be33952013-05-17 18:30:14 -04002814 mirror = io_bio->mirror_num;
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002815 if (likely(uptodate)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002816 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2817 page, start, end,
2818 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002819 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002820 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002821 else
Josef Bacik7870d082017-05-05 11:57:15 -04002822 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2823 failure_tree, tree, start,
2824 page,
2825 btrfs_ino(BTRFS_I(inode)), 0);
Chris Masond1310b22008-01-24 16:13:08 -05002826 }
Josef Bacikea466792012-03-26 21:57:36 -04002827
Miao Xief2a09da2013-07-25 19:22:33 +08002828 if (likely(uptodate))
2829 goto readpage_ok;
2830
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002831 if (data_inode) {
Liu Bo9d0d1c82017-03-24 15:04:50 -07002832
2833 /*
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002834 * The generic bio_readpage_error handles errors the
2835 * following way: If possible, new read requests are
2836 * created and submitted and will end up in
2837 * end_bio_extent_readpage as well (if we're lucky,
2838 * not in the !uptodate case). In that case it returns
2839 * 0 and we just go on with the next page in our bio.
2840 * If it can't handle the error it will return -EIO and
2841 * we remain responsible for that page.
Liu Bo9d0d1c82017-03-24 15:04:50 -07002842 */
Nikolay Borisov78e62c02018-11-22 10:17:49 +02002843 ret = bio_readpage_error(bio, offset, page, start, end,
2844 mirror);
2845 if (ret == 0) {
2846 uptodate = !bio->bi_status;
2847 offset += len;
2848 continue;
2849 }
2850 } else {
2851 struct extent_buffer *eb;
2852
2853 eb = (struct extent_buffer *)page->private;
2854 set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
2855 eb->read_mirror = mirror;
2856 atomic_dec(&eb->io_pages);
2857 if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD,
2858 &eb->bflags))
2859 btree_readahead_hook(eb, -EIO);
Chris Mason7e383262008-04-09 16:28:12 -04002860 }
Miao Xief2a09da2013-07-25 19:22:33 +08002861readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002862 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002863 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002864 pgoff_t end_index = i_size >> PAGE_SHIFT;
Liu Boa583c022014-08-19 23:32:22 +08002865 unsigned off;
Josef Bacika71754f2013-06-17 17:14:39 -04002866
2867 /* Zero out the end if this page straddles i_size */
Johannes Thumshirn70730172018-12-05 15:23:03 +01002868 off = offset_in_page(i_size);
Liu Boa583c022014-08-19 23:32:22 +08002869 if (page->index == end_index && off)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002870 zero_user_segment(page, off, PAGE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002871 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002872 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002873 ClearPageUptodate(page);
2874 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002875 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002876 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002877 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002878
2879 if (unlikely(!uptodate)) {
2880 if (extent_len) {
2881 endio_readpage_release_extent(tree,
2882 extent_start,
2883 extent_len, 1);
2884 extent_start = 0;
2885 extent_len = 0;
2886 }
2887 endio_readpage_release_extent(tree, start,
2888 end - start + 1, 0);
2889 } else if (!extent_len) {
2890 extent_start = start;
2891 extent_len = end + 1 - start;
2892 } else if (extent_start + extent_len == start) {
2893 extent_len += end + 1 - start;
2894 } else {
2895 endio_readpage_release_extent(tree, extent_start,
2896 extent_len, uptodate);
2897 extent_start = start;
2898 extent_len = end + 1 - start;
2899 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002900 }
Chris Masond1310b22008-01-24 16:13:08 -05002901
Miao Xie883d0de2013-07-25 19:22:35 +08002902 if (extent_len)
2903 endio_readpage_release_extent(tree, extent_start, extent_len,
2904 uptodate);
David Sterbab3a0dd52018-11-22 17:16:49 +01002905 btrfs_io_bio_free_csum(io_bio);
Chris Masond1310b22008-01-24 16:13:08 -05002906 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002907}
2908
Chris Mason9be33952013-05-17 18:30:14 -04002909/*
David Sterba184f9992017-06-12 17:29:39 +02002910 * Initialize the members up to but not including 'bio'. Use after allocating a
2911 * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2912 * 'bio' because use of __GFP_ZERO is not supported.
Chris Mason9be33952013-05-17 18:30:14 -04002913 */
David Sterba184f9992017-06-12 17:29:39 +02002914static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
Chris Masond1310b22008-01-24 16:13:08 -05002915{
David Sterba184f9992017-06-12 17:29:39 +02002916 memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2917}
2918
2919/*
David Sterba6e707bc2017-06-02 17:26:26 +02002920 * The following helpers allocate a bio. As it's backed by a bioset, it'll
2921 * never fail. We're returning a bio right now but you can call btrfs_io_bio
2922 * for the appropriate container_of magic
Chris Masond1310b22008-01-24 16:13:08 -05002923 */
David Sterbae749af442019-06-18 20:00:16 +02002924struct bio *btrfs_bio_alloc(u64 first_byte)
Chris Masond1310b22008-01-24 16:13:08 -05002925{
2926 struct bio *bio;
2927
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002928 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
David Sterbac821e7f32017-06-02 18:35:36 +02002929 bio->bi_iter.bi_sector = first_byte >> 9;
David Sterba184f9992017-06-12 17:29:39 +02002930 btrfs_io_bio_init(btrfs_io_bio(bio));
Chris Masond1310b22008-01-24 16:13:08 -05002931 return bio;
2932}
2933
David Sterba8b6c1d52017-06-02 17:48:13 +02002934struct bio *btrfs_bio_clone(struct bio *bio)
Chris Mason9be33952013-05-17 18:30:14 -04002935{
Miao Xie23ea8e52014-09-12 18:43:54 +08002936 struct btrfs_io_bio *btrfs_bio;
2937 struct bio *new;
Chris Mason9be33952013-05-17 18:30:14 -04002938
David Sterba6e707bc2017-06-02 17:26:26 +02002939 /* Bio allocation backed by a bioset does not fail */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002940 new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
David Sterba6e707bc2017-06-02 17:26:26 +02002941 btrfs_bio = btrfs_io_bio(new);
David Sterba184f9992017-06-12 17:29:39 +02002942 btrfs_io_bio_init(btrfs_bio);
David Sterba6e707bc2017-06-02 17:26:26 +02002943 btrfs_bio->iter = bio->bi_iter;
Miao Xie23ea8e52014-09-12 18:43:54 +08002944 return new;
2945}
Chris Mason9be33952013-05-17 18:30:14 -04002946
David Sterbac5e4c3d2017-06-12 17:29:41 +02002947struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
Chris Mason9be33952013-05-17 18:30:14 -04002948{
Miao Xiefacc8a222013-07-25 19:22:34 +08002949 struct bio *bio;
2950
David Sterba6e707bc2017-06-02 17:26:26 +02002951 /* Bio allocation backed by a bioset does not fail */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002952 bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
David Sterba184f9992017-06-12 17:29:39 +02002953 btrfs_io_bio_init(btrfs_io_bio(bio));
Miao Xiefacc8a222013-07-25 19:22:34 +08002954 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002955}
2956
Liu Boe4770942017-05-16 10:57:14 -07002957struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
Liu Bo2f8e9142017-05-15 17:43:31 -07002958{
2959 struct bio *bio;
2960 struct btrfs_io_bio *btrfs_bio;
2961
2962 /* this will never fail when it's backed by a bioset */
Kent Overstreet8ac9f7c2018-05-20 18:25:56 -04002963 bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
Liu Bo2f8e9142017-05-15 17:43:31 -07002964 ASSERT(bio);
2965
2966 btrfs_bio = btrfs_io_bio(bio);
David Sterba184f9992017-06-12 17:29:39 +02002967 btrfs_io_bio_init(btrfs_bio);
Liu Bo2f8e9142017-05-15 17:43:31 -07002968
2969 bio_trim(bio, offset >> 9, size >> 9);
Liu Bo17347ce2017-05-15 15:33:27 -07002970 btrfs_bio->iter = bio->bi_iter;
Liu Bo2f8e9142017-05-15 17:43:31 -07002971 return bio;
2972}
Chris Mason9be33952013-05-17 18:30:14 -04002973
David Sterba4b81ba42017-06-06 19:14:26 +02002974/*
2975 * @opf: bio REQ_OP_* and REQ_* flags as one value
David Sterbab8b3d622017-06-12 19:50:41 +02002976 * @wbc: optional writeback control for io accounting
2977 * @page: page to add to the bio
2978 * @pg_offset: offset of the new bio or to check whether we are adding
2979 * a contiguous page to the previous one
2980 * @size: portion of page that we want to write
2981 * @offset: starting offset in the page
David Sterba5c2b1fd2017-06-06 19:22:55 +02002982 * @bio_ret: must be valid pointer, newly allocated bio will be stored there
David Sterbab8b3d622017-06-12 19:50:41 +02002983 * @end_io_func: end_io callback for new bio
2984 * @mirror_num: desired mirror to read/write
2985 * @prev_bio_flags: flags of previous bio to see if we can merge the current one
2986 * @bio_flags: flags of the current bio to see if we can merge them
David Sterba4b81ba42017-06-06 19:14:26 +02002987 */
David Sterba0ceb34b2020-02-05 19:09:28 +01002988static int submit_extent_page(unsigned int opf,
Chris Masonda2f0f72015-07-02 13:57:22 -07002989 struct writeback_control *wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02002990 struct page *page, u64 offset,
David Sterba6c5a4e22017-10-04 17:10:34 +02002991 size_t size, unsigned long pg_offset,
Chris Masond1310b22008-01-24 16:13:08 -05002992 struct bio **bio_ret,
Chris Masonf1885912008-04-09 16:28:12 -04002993 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002994 int mirror_num,
2995 unsigned long prev_bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002996 unsigned long bio_flags,
2997 bool force_bio_submit)
Chris Masond1310b22008-01-24 16:13:08 -05002998{
2999 int ret = 0;
3000 struct bio *bio;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003001 size_t page_size = min_t(size_t, size, PAGE_SIZE);
David Sterba6273b7f2017-10-04 17:30:11 +02003002 sector_t sector = offset >> 9;
David Sterba0ceb34b2020-02-05 19:09:28 +01003003 struct extent_io_tree *tree = &BTRFS_I(page->mapping->host)->io_tree;
Chris Masond1310b22008-01-24 16:13:08 -05003004
David Sterba5c2b1fd2017-06-06 19:22:55 +02003005 ASSERT(bio_ret);
3006
3007 if (*bio_ret) {
David Sterba0c8508a2017-06-12 20:00:43 +02003008 bool contig;
3009 bool can_merge = true;
3010
Chris Masond1310b22008-01-24 16:13:08 -05003011 bio = *bio_ret;
David Sterba0c8508a2017-06-12 20:00:43 +02003012 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
Kent Overstreet4f024f32013-10-11 15:44:27 -07003013 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04003014 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07003015 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04003016
Nikolay Borisovda12fe52018-11-27 20:57:58 +02003017 ASSERT(tree->ops);
3018 if (btrfs_bio_fits_in_stripe(page, page_size, bio, bio_flags))
David Sterba0c8508a2017-06-12 20:00:43 +02003019 can_merge = false;
3020
3021 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
Filipe Manana005efed2015-09-14 09:09:31 +01003022 force_bio_submit ||
David Sterba6c5a4e22017-10-04 17:10:34 +02003023 bio_add_page(bio, page, page_size, pg_offset) < page_size) {
Mike Christie1f7ad752016-06-05 14:31:51 -05003024 ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
Naohiro Aota289454a2015-01-06 01:01:03 +09003025 if (ret < 0) {
3026 *bio_ret = NULL;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003027 return ret;
Naohiro Aota289454a2015-01-06 01:01:03 +09003028 }
Chris Masond1310b22008-01-24 16:13:08 -05003029 bio = NULL;
3030 } else {
Chris Masonda2f0f72015-07-02 13:57:22 -07003031 if (wbc)
Tejun Heo34e51a52019-06-27 13:39:49 -07003032 wbc_account_cgroup_owner(wbc, page, page_size);
Chris Masond1310b22008-01-24 16:13:08 -05003033 return 0;
3034 }
3035 }
Chris Masonc8b97812008-10-29 14:49:59 -04003036
David Sterbae749af442019-06-18 20:00:16 +02003037 bio = btrfs_bio_alloc(offset);
David Sterba6c5a4e22017-10-04 17:10:34 +02003038 bio_add_page(bio, page, page_size, pg_offset);
Chris Masond1310b22008-01-24 16:13:08 -05003039 bio->bi_end_io = end_io_func;
3040 bio->bi_private = tree;
Jens Axboee6959b92017-06-27 11:51:28 -06003041 bio->bi_write_hint = page->mapping->host->i_write_hint;
David Sterba4b81ba42017-06-06 19:14:26 +02003042 bio->bi_opf = opf;
Chris Masonda2f0f72015-07-02 13:57:22 -07003043 if (wbc) {
David Sterba429aebc2019-11-18 23:27:55 +01003044 struct block_device *bdev;
3045
3046 bdev = BTRFS_I(page->mapping->host)->root->fs_info->fs_devices->latest_bdev;
3047 bio_set_dev(bio, bdev);
Chris Masonda2f0f72015-07-02 13:57:22 -07003048 wbc_init_bio(wbc, bio);
Tejun Heo34e51a52019-06-27 13:39:49 -07003049 wbc_account_cgroup_owner(wbc, page, page_size);
Chris Masonda2f0f72015-07-02 13:57:22 -07003050 }
Chris Mason70dec802008-01-29 09:59:12 -05003051
David Sterba5c2b1fd2017-06-06 19:22:55 +02003052 *bio_ret = bio;
Chris Masond1310b22008-01-24 16:13:08 -05003053
3054 return ret;
3055}
3056
Eric Sandeen48a3b632013-04-25 20:41:01 +00003057static void attach_extent_buffer_page(struct extent_buffer *eb,
3058 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05003059{
3060 if (!PagePrivate(page)) {
3061 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003062 get_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05003063 set_page_private(page, (unsigned long)eb);
3064 } else {
3065 WARN_ON(page->private != (unsigned long)eb);
3066 }
3067}
3068
Chris Masond1310b22008-01-24 16:13:08 -05003069void set_page_extent_mapped(struct page *page)
3070{
3071 if (!PagePrivate(page)) {
3072 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003073 get_page(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04003074 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05003075 }
3076}
3077
Miao Xie125bac012013-07-25 19:22:37 +08003078static struct extent_map *
3079__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
3080 u64 start, u64 len, get_extent_t *get_extent,
3081 struct extent_map **em_cached)
3082{
3083 struct extent_map *em;
3084
3085 if (em_cached && *em_cached) {
3086 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00003087 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08003088 start < extent_map_end(em)) {
Elena Reshetova490b54d2017-03-03 10:55:12 +02003089 refcount_inc(&em->refs);
Miao Xie125bac012013-07-25 19:22:37 +08003090 return em;
3091 }
3092
3093 free_extent_map(em);
3094 *em_cached = NULL;
3095 }
3096
Omar Sandoval39b07b52019-12-02 17:34:23 -08003097 em = get_extent(BTRFS_I(inode), page, pg_offset, start, len);
Miao Xie125bac012013-07-25 19:22:37 +08003098 if (em_cached && !IS_ERR_OR_NULL(em)) {
3099 BUG_ON(*em_cached);
Elena Reshetova490b54d2017-03-03 10:55:12 +02003100 refcount_inc(&em->refs);
Miao Xie125bac012013-07-25 19:22:37 +08003101 *em_cached = em;
3102 }
3103 return em;
3104}
Chris Masond1310b22008-01-24 16:13:08 -05003105/*
3106 * basic readpage implementation. Locked extent state structs are inserted
3107 * into the tree that are removed when the IO is done (by the end_io
3108 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003109 * XXX JDM: This needs looking at to ensure proper page locking
Liu Bobaf863b2016-07-11 10:39:07 -07003110 * return 0 on success, otherwise return error
Chris Masond1310b22008-01-24 16:13:08 -05003111 */
Miao Xie99740902013-07-25 19:22:36 +08003112static int __do_readpage(struct extent_io_tree *tree,
3113 struct page *page,
3114 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003115 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003116 struct bio **bio, int mirror_num,
David Sterbaf1c77c52017-06-06 19:03:49 +02003117 unsigned long *bio_flags, unsigned int read_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01003118 u64 *prev_em_start)
Chris Masond1310b22008-01-24 16:13:08 -05003119{
3120 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003121 u64 start = page_offset(page);
David Sterba8eec8292017-06-06 19:50:13 +02003122 const u64 end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05003123 u64 cur = start;
3124 u64 extent_offset;
3125 u64 last_byte = i_size_read(inode);
3126 u64 block_start;
3127 u64 cur_end;
Chris Masond1310b22008-01-24 16:13:08 -05003128 struct extent_map *em;
Liu Bobaf863b2016-07-11 10:39:07 -07003129 int ret = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003130 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02003131 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003132 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04003133 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05003134 size_t blocksize = inode->i_sb->s_blocksize;
Filipe Manana7f042a82016-01-27 19:17:20 +00003135 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003136
David Sterbaae6957e2020-02-05 19:09:30 +01003137 ASSERT(tree == &BTRFS_I(inode)->io_tree);
3138
Chris Masond1310b22008-01-24 16:13:08 -05003139 set_page_extent_mapped(page);
3140
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003141 if (!PageUptodate(page)) {
3142 if (cleancache_get_page(page) == 0) {
3143 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08003144 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003145 goto out;
3146 }
3147 }
3148
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003149 if (page->index == last_byte >> PAGE_SHIFT) {
Chris Masonc8b97812008-10-29 14:49:59 -04003150 char *userpage;
Johannes Thumshirn70730172018-12-05 15:23:03 +01003151 size_t zero_offset = offset_in_page(last_byte);
Chris Masonc8b97812008-10-29 14:49:59 -04003152
3153 if (zero_offset) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003154 iosize = PAGE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08003155 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04003156 memset(userpage + zero_offset, 0, iosize);
3157 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003158 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04003159 }
3160 }
Chris Masond1310b22008-01-24 16:13:08 -05003161 while (cur <= end) {
Filipe Manana005efed2015-09-14 09:09:31 +01003162 bool force_bio_submit = false;
David Sterba6273b7f2017-10-04 17:30:11 +02003163 u64 offset;
Josef Bacikc8f2f242013-02-11 11:33:00 -05003164
Chris Masond1310b22008-01-24 16:13:08 -05003165 if (cur >= last_byte) {
3166 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003167 struct extent_state *cached = NULL;
3168
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003169 iosize = PAGE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08003170 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003171 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003172 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003173 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003174 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003175 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00003176 unlock_extent_cached(tree, cur,
David Sterbae43bbe52017-12-12 21:43:52 +01003177 cur + iosize - 1, &cached);
Chris Masond1310b22008-01-24 16:13:08 -05003178 break;
3179 }
Miao Xie125bac012013-07-25 19:22:37 +08003180 em = __get_extent_map(inode, page, pg_offset, cur,
3181 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02003182 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003183 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003184 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003185 break;
3186 }
Chris Masond1310b22008-01-24 16:13:08 -05003187 extent_offset = cur - em->start;
3188 BUG_ON(extent_map_end(em) <= cur);
3189 BUG_ON(end < cur);
3190
Li Zefan261507a02010-12-17 14:21:50 +08003191 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07003192 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08003193 extent_set_compress_type(&this_bio_flag,
3194 em->compress_type);
3195 }
Chris Masonc8b97812008-10-29 14:49:59 -04003196
Chris Masond1310b22008-01-24 16:13:08 -05003197 iosize = min(extent_map_end(em) - cur, end - cur + 1);
3198 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00003199 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04003200 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
3201 disk_io_size = em->block_len;
David Sterba6273b7f2017-10-04 17:30:11 +02003202 offset = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003203 } else {
David Sterba6273b7f2017-10-04 17:30:11 +02003204 offset = em->block_start + extent_offset;
Chris Masonc8b97812008-10-29 14:49:59 -04003205 disk_io_size = iosize;
3206 }
Chris Masond1310b22008-01-24 16:13:08 -05003207 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04003208 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3209 block_start = EXTENT_MAP_HOLE;
Filipe Manana005efed2015-09-14 09:09:31 +01003210
3211 /*
3212 * If we have a file range that points to a compressed extent
3213 * and it's followed by a consecutive file range that points to
3214 * to the same compressed extent (possibly with a different
3215 * offset and/or length, so it either points to the whole extent
3216 * or only part of it), we must make sure we do not submit a
3217 * single bio to populate the pages for the 2 ranges because
3218 * this makes the compressed extent read zero out the pages
3219 * belonging to the 2nd range. Imagine the following scenario:
3220 *
3221 * File layout
3222 * [0 - 8K] [8K - 24K]
3223 * | |
3224 * | |
3225 * points to extent X, points to extent X,
3226 * offset 4K, length of 8K offset 0, length 16K
3227 *
3228 * [extent X, compressed length = 4K uncompressed length = 16K]
3229 *
3230 * If the bio to read the compressed extent covers both ranges,
3231 * it will decompress extent X into the pages belonging to the
3232 * first range and then it will stop, zeroing out the remaining
3233 * pages that belong to the other range that points to extent X.
3234 * So here we make sure we submit 2 bios, one for the first
3235 * range and another one for the third range. Both will target
3236 * the same physical extent from disk, but we can't currently
3237 * make the compressed bio endio callback populate the pages
3238 * for both ranges because each compressed bio is tightly
3239 * coupled with a single extent map, and each range can have
3240 * an extent map with a different offset value relative to the
3241 * uncompressed data of our extent and different lengths. This
3242 * is a corner case so we prioritize correctness over
3243 * non-optimal behavior (submitting 2 bios for the same extent).
3244 */
3245 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3246 prev_em_start && *prev_em_start != (u64)-1 &&
Filipe Manana8e928212019-02-14 15:17:20 +00003247 *prev_em_start != em->start)
Filipe Manana005efed2015-09-14 09:09:31 +01003248 force_bio_submit = true;
3249
3250 if (prev_em_start)
Filipe Manana8e928212019-02-14 15:17:20 +00003251 *prev_em_start = em->start;
Filipe Manana005efed2015-09-14 09:09:31 +01003252
Chris Masond1310b22008-01-24 16:13:08 -05003253 free_extent_map(em);
3254 em = NULL;
3255
3256 /* we've found a hole, just zero and go on */
3257 if (block_start == EXTENT_MAP_HOLE) {
3258 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003259 struct extent_state *cached = NULL;
3260
Cong Wang7ac687d2011-11-25 23:14:28 +08003261 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003262 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003263 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003264 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003265
3266 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003267 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00003268 unlock_extent_cached(tree, cur,
David Sterbae43bbe52017-12-12 21:43:52 +01003269 cur + iosize - 1, &cached);
Chris Masond1310b22008-01-24 16:13:08 -05003270 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003271 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003272 continue;
3273 }
3274 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04003275 if (test_range_bit(tree, cur, cur_end,
3276 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04003277 check_page_uptodate(tree, page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003278 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003279 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003280 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003281 continue;
3282 }
Chris Mason70dec802008-01-29 09:59:12 -05003283 /* we have an inline extent but it didn't get marked up
3284 * to date. Error out
3285 */
3286 if (block_start == EXTENT_MAP_INLINE) {
3287 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003288 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05003289 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003290 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05003291 continue;
3292 }
Chris Masond1310b22008-01-24 16:13:08 -05003293
David Sterba0ceb34b2020-02-05 19:09:28 +01003294 ret = submit_extent_page(REQ_OP_READ | read_flags, NULL,
David Sterba6273b7f2017-10-04 17:30:11 +02003295 page, offset, disk_io_size,
David Sterbafa17ed02019-10-03 17:29:05 +02003296 pg_offset, bio,
Chris Masonc8b97812008-10-29 14:49:59 -04003297 end_bio_extent_readpage, mirror_num,
3298 *bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01003299 this_bio_flag,
3300 force_bio_submit);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003301 if (!ret) {
3302 nr++;
3303 *bio_flags = this_bio_flag;
3304 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003305 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003306 unlock_extent(tree, cur, cur + iosize - 1);
Liu Bobaf863b2016-07-11 10:39:07 -07003307 goto out;
Josef Bacikedd33c92012-10-05 16:40:32 -04003308 }
Chris Masond1310b22008-01-24 16:13:08 -05003309 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003310 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003311 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003312out:
Chris Masond1310b22008-01-24 16:13:08 -05003313 if (!nr) {
3314 if (!PageError(page))
3315 SetPageUptodate(page);
3316 unlock_page(page);
3317 }
Liu Bobaf863b2016-07-11 10:39:07 -07003318 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05003319}
3320
Nikolay Borisove65ef212019-03-11 09:55:38 +02003321static inline void contiguous_readpages(struct extent_io_tree *tree,
Miao Xie99740902013-07-25 19:22:36 +08003322 struct page *pages[], int nr_pages,
3323 u64 start, u64 end,
Miao Xie125bac012013-07-25 19:22:37 +08003324 struct extent_map **em_cached,
Nikolay Borisovd3fac6b2017-10-24 11:50:39 +03003325 struct bio **bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003326 unsigned long *bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003327 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003328{
Nikolay Borisov23d31bd2019-05-07 10:19:23 +03003329 struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host);
Miao Xie99740902013-07-25 19:22:36 +08003330 int index;
3331
David Sterbaae6957e2020-02-05 19:09:30 +01003332 ASSERT(tree == &inode->io_tree);
3333
Nikolay Borisov23d31bd2019-05-07 10:19:23 +03003334 btrfs_lock_and_flush_ordered_range(tree, inode, start, end, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003335
3336 for (index = 0; index < nr_pages; index++) {
David Sterba4ef77692017-06-23 04:09:57 +02003337 __do_readpage(tree, pages[index], btrfs_get_extent, em_cached,
Jens Axboe5e9d3982018-08-17 15:45:39 -07003338 bio, 0, bio_flags, REQ_RAHEAD, prev_em_start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003339 put_page(pages[index]);
Miao Xie99740902013-07-25 19:22:36 +08003340 }
3341}
3342
Miao Xie99740902013-07-25 19:22:36 +08003343static int __extent_read_full_page(struct extent_io_tree *tree,
3344 struct page *page,
3345 get_extent_t *get_extent,
3346 struct bio **bio, int mirror_num,
David Sterbaf1c77c52017-06-06 19:03:49 +02003347 unsigned long *bio_flags,
3348 unsigned int read_flags)
Miao Xie99740902013-07-25 19:22:36 +08003349{
Nikolay Borisov23d31bd2019-05-07 10:19:23 +03003350 struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
Miao Xie99740902013-07-25 19:22:36 +08003351 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003352 u64 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003353 int ret;
3354
David Sterbaae6957e2020-02-05 19:09:30 +01003355 ASSERT(tree == &inode->io_tree);
3356
Nikolay Borisov23d31bd2019-05-07 10:19:23 +03003357 btrfs_lock_and_flush_ordered_range(tree, inode, start, end, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003358
Miao Xie125bac012013-07-25 19:22:37 +08003359 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003360 bio_flags, read_flags, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003361 return ret;
3362}
3363
Chris Masond1310b22008-01-24 16:13:08 -05003364int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003365 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003366{
3367 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003368 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003369 int ret;
3370
David Sterbaae6957e2020-02-05 19:09:30 +01003371 ASSERT(tree == &BTRFS_I(page->mapping->host)->io_tree);
3372
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003373 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003374 &bio_flags, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003375 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05003376 ret = submit_one_bio(bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003377 return ret;
3378}
Chris Masond1310b22008-01-24 16:13:08 -05003379
David Sterba3d4b9492017-02-10 19:33:41 +01003380static void update_nr_written(struct writeback_control *wbc,
Liu Boa91326672016-03-07 16:56:21 -08003381 unsigned long nr_written)
Chris Mason11c83492009-04-20 15:50:09 -04003382{
3383 wbc->nr_to_write -= nr_written;
Chris Mason11c83492009-04-20 15:50:09 -04003384}
3385
Chris Masond1310b22008-01-24 16:13:08 -05003386/*
Chris Mason40f76582014-05-21 13:35:51 -07003387 * helper for __extent_writepage, doing all of the delayed allocation setup.
3388 *
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003389 * This returns 1 if btrfs_run_delalloc_range function did all the work required
Chris Mason40f76582014-05-21 13:35:51 -07003390 * to write the page (copy into inline extent). In this case the IO has
3391 * been started and the page is already unlocked.
3392 *
3393 * This returns 0 if all went well (page still locked)
3394 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003395 */
Chris Mason40f76582014-05-21 13:35:51 -07003396static noinline_for_stack int writepage_delalloc(struct inode *inode,
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003397 struct page *page, struct writeback_control *wbc,
3398 u64 delalloc_start, unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003399{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003400 u64 page_end = delalloc_start + PAGE_SIZE - 1;
Lu Fengqi3522e902018-11-29 11:33:38 +08003401 bool found;
Chris Mason40f76582014-05-21 13:35:51 -07003402 u64 delalloc_to_write = 0;
3403 u64 delalloc_end = 0;
3404 int ret;
3405 int page_started = 0;
3406
Chris Mason40f76582014-05-21 13:35:51 -07003407
3408 while (delalloc_end < page_end) {
Goldwyn Rodrigues99780592019-06-21 10:02:54 -05003409 found = find_lock_delalloc_range(inode, page,
Chris Mason40f76582014-05-21 13:35:51 -07003410 &delalloc_start,
Nikolay Borisov917aace2018-10-26 14:43:20 +03003411 &delalloc_end);
Lu Fengqi3522e902018-11-29 11:33:38 +08003412 if (!found) {
Chris Mason40f76582014-05-21 13:35:51 -07003413 delalloc_start = delalloc_end + 1;
3414 continue;
3415 }
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003416 ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
3417 delalloc_end, &page_started, nr_written, wbc);
Chris Mason40f76582014-05-21 13:35:51 -07003418 if (ret) {
3419 SetPageError(page);
Nikolay Borisov5eaad972018-11-01 14:09:46 +02003420 /*
3421 * btrfs_run_delalloc_range should return < 0 for error
3422 * but just in case, we use > 0 here meaning the IO is
3423 * started, so we don't want to return > 0 unless
3424 * things are going well.
Chris Mason40f76582014-05-21 13:35:51 -07003425 */
3426 ret = ret < 0 ? ret : -EIO;
3427 goto done;
3428 }
3429 /*
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003430 * delalloc_end is already one less than the total length, so
3431 * we don't subtract one from PAGE_SIZE
Chris Mason40f76582014-05-21 13:35:51 -07003432 */
3433 delalloc_to_write += (delalloc_end - delalloc_start +
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003434 PAGE_SIZE) >> PAGE_SHIFT;
Chris Mason40f76582014-05-21 13:35:51 -07003435 delalloc_start = delalloc_end + 1;
3436 }
3437 if (wbc->nr_to_write < delalloc_to_write) {
3438 int thresh = 8192;
3439
3440 if (delalloc_to_write < thresh * 2)
3441 thresh = delalloc_to_write;
3442 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3443 thresh);
3444 }
3445
3446 /* did the fill delalloc function already unlock and start
3447 * the IO?
3448 */
3449 if (page_started) {
3450 /*
3451 * we've unlocked the page, so we can't update
3452 * the mapping's writeback index, just update
3453 * nr_to_write.
3454 */
3455 wbc->nr_to_write -= *nr_written;
3456 return 1;
3457 }
3458
3459 ret = 0;
3460
3461done:
3462 return ret;
3463}
3464
3465/*
3466 * helper for __extent_writepage. This calls the writepage start hooks,
3467 * and does the loop to map the page into extents and bios.
3468 *
3469 * We return 1 if the IO is started and the page is unlocked,
3470 * 0 if all went well (page still locked)
3471 * < 0 if there were errors (page still locked)
3472 */
3473static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3474 struct page *page,
3475 struct writeback_control *wbc,
3476 struct extent_page_data *epd,
3477 loff_t i_size,
3478 unsigned long nr_written,
David Sterba57e5ffe2019-10-29 18:28:55 +01003479 int *nr_ret)
Chris Mason40f76582014-05-21 13:35:51 -07003480{
David Sterba45b08402020-02-05 19:09:26 +01003481 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003482 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003483 u64 page_end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05003484 u64 end;
3485 u64 cur = start;
3486 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003487 u64 block_start;
3488 u64 iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003489 struct extent_map *em;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003490 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003491 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003492 int ret = 0;
3493 int nr = 0;
David Sterba57e5ffe2019-10-29 18:28:55 +01003494 const unsigned int write_flags = wbc_to_write_flags(wbc);
Chris Mason40f76582014-05-21 13:35:51 -07003495 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003496
Nikolay Borisovd75855b2018-11-01 14:09:47 +02003497 ret = btrfs_writepage_cow_fixup(page, start, page_end);
3498 if (ret) {
3499 /* Fixup worker will requeue */
Josef Bacik5ab58052020-01-21 11:51:43 -05003500 redirty_page_for_writepage(wbc, page);
Nikolay Borisovd75855b2018-11-01 14:09:47 +02003501 update_nr_written(wbc, nr_written);
3502 unlock_page(page);
3503 return 1;
Chris Mason247e7432008-07-17 12:53:51 -04003504 }
3505
Chris Mason11c83492009-04-20 15:50:09 -04003506 /*
3507 * we don't want to touch the inode after unlocking the page,
3508 * so we update the mapping writeback index now
3509 */
David Sterba3d4b9492017-02-10 19:33:41 +01003510 update_nr_written(wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003511
Chris Masond1310b22008-01-24 16:13:08 -05003512 end = page_end;
Chris Masond1310b22008-01-24 16:13:08 -05003513 blocksize = inode->i_sb->s_blocksize;
3514
3515 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003516 u64 em_end;
David Sterba6273b7f2017-10-04 17:30:11 +02003517 u64 offset;
David Sterba58409ed2016-05-04 11:46:10 +02003518
Chris Mason40f76582014-05-21 13:35:51 -07003519 if (cur >= i_size) {
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02003520 btrfs_writepage_endio_finish_ordered(page, cur,
Nikolay Borisovc6297322018-11-08 10:18:08 +02003521 page_end, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003522 break;
3523 }
Omar Sandoval39b07b52019-12-02 17:34:23 -08003524 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur,
3525 end - cur + 1);
David Sterbac7040052011-04-19 18:00:01 +02003526 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003527 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003528 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003529 break;
3530 }
3531
3532 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003533 em_end = extent_map_end(em);
3534 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003535 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003536 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003537 iosize = ALIGN(iosize, blocksize);
David Sterba6273b7f2017-10-04 17:30:11 +02003538 offset = em->block_start + extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003539 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003540 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003541 free_extent_map(em);
3542 em = NULL;
3543
Chris Masonc8b97812008-10-29 14:49:59 -04003544 /*
3545 * compressed and inline extents are written through other
3546 * paths in the FS
3547 */
3548 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003549 block_start == EXTENT_MAP_INLINE) {
Omar Sandovalc8b04032019-12-02 17:34:24 -08003550 if (compressed)
Chris Masonc8b97812008-10-29 14:49:59 -04003551 nr++;
Omar Sandovalc8b04032019-12-02 17:34:24 -08003552 else
3553 btrfs_writepage_endio_finish_ordered(page, cur,
3554 cur + iosize - 1, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003555 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003556 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003557 continue;
3558 }
Chris Masonc8b97812008-10-29 14:49:59 -04003559
David Sterba5cdc84b2018-07-18 20:32:52 +02003560 btrfs_set_range_writeback(tree, cur, cur + iosize - 1);
David Sterba58409ed2016-05-04 11:46:10 +02003561 if (!PageWriteback(page)) {
3562 btrfs_err(BTRFS_I(inode)->root->fs_info,
3563 "page %lu not writeback, cur %llu end %llu",
3564 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003565 }
David Sterba58409ed2016-05-04 11:46:10 +02003566
David Sterba0ceb34b2020-02-05 19:09:28 +01003567 ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc,
David Sterba6273b7f2017-10-04 17:30:11 +02003568 page, offset, iosize, pg_offset,
David Sterbafa17ed02019-10-03 17:29:05 +02003569 &epd->bio,
David Sterba58409ed2016-05-04 11:46:10 +02003570 end_bio_extent_writepage,
3571 0, 0, 0, false);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003572 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003573 SetPageError(page);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003574 if (PageWriteback(page))
3575 end_page_writeback(page);
3576 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003577
Chris Masond1310b22008-01-24 16:13:08 -05003578 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003579 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003580 nr++;
3581 }
Chris Mason40f76582014-05-21 13:35:51 -07003582 *nr_ret = nr;
Chris Mason40f76582014-05-21 13:35:51 -07003583 return ret;
3584}
3585
3586/*
3587 * the writepage semantics are similar to regular writepage. extent
3588 * records are inserted to lock ranges in the tree, and as dirty areas
3589 * are found, they are marked writeback. Then the lock bits are removed
3590 * and the end_io handler clears the writeback ranges
Qu Wenruo30659762019-03-20 14:27:42 +08003591 *
3592 * Return 0 if everything goes well.
3593 * Return <0 for error.
Chris Mason40f76582014-05-21 13:35:51 -07003594 */
3595static int __extent_writepage(struct page *page, struct writeback_control *wbc,
David Sterbaaab6e9e2017-11-30 18:00:02 +01003596 struct extent_page_data *epd)
Chris Mason40f76582014-05-21 13:35:51 -07003597{
3598 struct inode *inode = page->mapping->host;
Chris Mason40f76582014-05-21 13:35:51 -07003599 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003600 u64 page_end = start + PAGE_SIZE - 1;
Chris Mason40f76582014-05-21 13:35:51 -07003601 int ret;
3602 int nr = 0;
Omar Sandovaleb70d222019-12-02 17:34:20 -08003603 size_t pg_offset;
Chris Mason40f76582014-05-21 13:35:51 -07003604 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003605 unsigned long end_index = i_size >> PAGE_SHIFT;
Chris Mason40f76582014-05-21 13:35:51 -07003606 unsigned long nr_written = 0;
3607
Chris Mason40f76582014-05-21 13:35:51 -07003608 trace___extent_writepage(page, inode, wbc);
3609
3610 WARN_ON(!PageLocked(page));
3611
3612 ClearPageError(page);
3613
Johannes Thumshirn70730172018-12-05 15:23:03 +01003614 pg_offset = offset_in_page(i_size);
Chris Mason40f76582014-05-21 13:35:51 -07003615 if (page->index > end_index ||
3616 (page->index == end_index && !pg_offset)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003617 page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
Chris Mason40f76582014-05-21 13:35:51 -07003618 unlock_page(page);
3619 return 0;
3620 }
3621
3622 if (page->index == end_index) {
3623 char *userpage;
3624
3625 userpage = kmap_atomic(page);
3626 memset(userpage + pg_offset, 0,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003627 PAGE_SIZE - pg_offset);
Chris Mason40f76582014-05-21 13:35:51 -07003628 kunmap_atomic(userpage);
3629 flush_dcache_page(page);
3630 }
3631
Chris Mason40f76582014-05-21 13:35:51 -07003632 set_page_extent_mapped(page);
3633
Nikolay Borisov7789a552018-11-08 10:18:06 +02003634 if (!epd->extent_locked) {
Nikolay Borisov8cc02372018-11-08 10:18:07 +02003635 ret = writepage_delalloc(inode, page, wbc, start, &nr_written);
Nikolay Borisov7789a552018-11-08 10:18:06 +02003636 if (ret == 1)
Omar Sandoval169d2c82019-12-02 17:34:21 -08003637 return 0;
Nikolay Borisov7789a552018-11-08 10:18:06 +02003638 if (ret)
3639 goto done;
3640 }
Chris Mason40f76582014-05-21 13:35:51 -07003641
3642 ret = __extent_writepage_io(inode, page, wbc, epd,
David Sterba57e5ffe2019-10-29 18:28:55 +01003643 i_size, nr_written, &nr);
Chris Mason40f76582014-05-21 13:35:51 -07003644 if (ret == 1)
Omar Sandoval169d2c82019-12-02 17:34:21 -08003645 return 0;
Chris Mason40f76582014-05-21 13:35:51 -07003646
3647done:
Chris Masond1310b22008-01-24 16:13:08 -05003648 if (nr == 0) {
3649 /* make sure the mapping tag for page dirty gets cleared */
3650 set_page_writeback(page);
3651 end_page_writeback(page);
3652 }
Filipe Manana61391d52014-05-09 17:17:40 +01003653 if (PageError(page)) {
3654 ret = ret < 0 ? ret : -EIO;
3655 end_extent_writepage(page, ret, start, page_end);
3656 }
Chris Masond1310b22008-01-24 16:13:08 -05003657 unlock_page(page);
Qu Wenruo30659762019-03-20 14:27:42 +08003658 ASSERT(ret <= 0);
Chris Mason40f76582014-05-21 13:35:51 -07003659 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05003660}
3661
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003662void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003663{
NeilBrown74316202014-07-07 15:16:04 +10003664 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3665 TASK_UNINTERRUPTIBLE);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003666}
3667
Filipe Manana18dfa712019-09-11 17:42:00 +01003668static void end_extent_buffer_writeback(struct extent_buffer *eb)
3669{
3670 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3671 smp_mb__after_atomic();
3672 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3673}
3674
Qu Wenruo2e3c2512019-03-20 14:27:46 +08003675/*
3676 * Lock eb pages and flush the bio if we can't the locks
3677 *
3678 * Return 0 if nothing went wrong
3679 * Return >0 is same as 0, except bio is not submitted
3680 * Return <0 if something went wrong, no page is locked
3681 */
David Sterba9df76fb2019-03-20 11:21:41 +01003682static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb,
Chris Mason0e378df2014-05-19 20:55:27 -07003683 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003684{
David Sterba9df76fb2019-03-20 11:21:41 +01003685 struct btrfs_fs_info *fs_info = eb->fs_info;
Qu Wenruo2e3c2512019-03-20 14:27:46 +08003686 int i, num_pages, failed_page_nr;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003687 int flush = 0;
3688 int ret = 0;
3689
3690 if (!btrfs_try_tree_write_lock(eb)) {
Qu Wenruof4340622019-03-20 14:27:41 +08003691 ret = flush_write_bio(epd);
Qu Wenruo2e3c2512019-03-20 14:27:46 +08003692 if (ret < 0)
3693 return ret;
3694 flush = 1;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003695 btrfs_tree_lock(eb);
3696 }
3697
3698 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3699 btrfs_tree_unlock(eb);
3700 if (!epd->sync_io)
3701 return 0;
3702 if (!flush) {
Qu Wenruof4340622019-03-20 14:27:41 +08003703 ret = flush_write_bio(epd);
Qu Wenruo2e3c2512019-03-20 14:27:46 +08003704 if (ret < 0)
3705 return ret;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003706 flush = 1;
3707 }
Chris Masona098d8e82012-03-21 12:09:56 -04003708 while (1) {
3709 wait_on_extent_buffer_writeback(eb);
3710 btrfs_tree_lock(eb);
3711 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3712 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003713 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003714 }
3715 }
3716
Josef Bacik51561ff2012-07-20 16:25:24 -04003717 /*
3718 * We need to do this to prevent races in people who check if the eb is
3719 * under IO since we can end up having no IO bits set for a short period
3720 * of time.
3721 */
3722 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003723 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3724 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003725 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003726 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Nikolay Borisov104b4e52017-06-20 21:01:20 +03003727 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
3728 -eb->len,
3729 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003730 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003731 } else {
3732 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003733 }
3734
3735 btrfs_tree_unlock(eb);
3736
3737 if (!ret)
3738 return ret;
3739
David Sterba65ad0102018-06-29 10:56:49 +02003740 num_pages = num_extent_pages(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003741 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003742 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003743
3744 if (!trylock_page(p)) {
3745 if (!flush) {
Filipe Manana18dfa712019-09-11 17:42:00 +01003746 int err;
3747
3748 err = flush_write_bio(epd);
3749 if (err < 0) {
3750 ret = err;
Qu Wenruo2e3c2512019-03-20 14:27:46 +08003751 failed_page_nr = i;
3752 goto err_unlock;
3753 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003754 flush = 1;
3755 }
3756 lock_page(p);
3757 }
3758 }
3759
3760 return ret;
Qu Wenruo2e3c2512019-03-20 14:27:46 +08003761err_unlock:
3762 /* Unlock already locked pages */
3763 for (i = 0; i < failed_page_nr; i++)
3764 unlock_page(eb->pages[i]);
Filipe Manana18dfa712019-09-11 17:42:00 +01003765 /*
3766 * Clear EXTENT_BUFFER_WRITEBACK and wake up anyone waiting on it.
3767 * Also set back EXTENT_BUFFER_DIRTY so future attempts to this eb can
3768 * be made and undo everything done before.
3769 */
3770 btrfs_tree_lock(eb);
3771 spin_lock(&eb->refs_lock);
3772 set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3773 end_extent_buffer_writeback(eb);
3774 spin_unlock(&eb->refs_lock);
3775 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, eb->len,
3776 fs_info->dirty_metadata_batch);
3777 btrfs_clear_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3778 btrfs_tree_unlock(eb);
Qu Wenruo2e3c2512019-03-20 14:27:46 +08003779 return ret;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003780}
3781
Filipe Manana656f30d2014-09-26 12:25:56 +01003782static void set_btree_ioerr(struct page *page)
3783{
3784 struct extent_buffer *eb = (struct extent_buffer *)page->private;
Dennis Zhoueb5b64f2019-09-13 14:54:07 +01003785 struct btrfs_fs_info *fs_info;
Filipe Manana656f30d2014-09-26 12:25:56 +01003786
3787 SetPageError(page);
3788 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3789 return;
3790
3791 /*
Dennis Zhoueb5b64f2019-09-13 14:54:07 +01003792 * If we error out, we should add back the dirty_metadata_bytes
3793 * to make it consistent.
3794 */
3795 fs_info = eb->fs_info;
3796 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
3797 eb->len, fs_info->dirty_metadata_batch);
3798
3799 /*
Filipe Manana656f30d2014-09-26 12:25:56 +01003800 * If writeback for a btree extent that doesn't belong to a log tree
3801 * failed, increment the counter transaction->eb_write_errors.
3802 * We do this because while the transaction is running and before it's
3803 * committing (when we call filemap_fdata[write|wait]_range against
3804 * the btree inode), we might have
3805 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3806 * returns an error or an error happens during writeback, when we're
3807 * committing the transaction we wouldn't know about it, since the pages
3808 * can be no longer dirty nor marked anymore for writeback (if a
3809 * subsequent modification to the extent buffer didn't happen before the
3810 * transaction commit), which makes filemap_fdata[write|wait]_range not
3811 * able to find the pages tagged with SetPageError at transaction
3812 * commit time. So if this happens we must abort the transaction,
3813 * otherwise we commit a super block with btree roots that point to
3814 * btree nodes/leafs whose content on disk is invalid - either garbage
3815 * or the content of some node/leaf from a past generation that got
3816 * cowed or deleted and is no longer valid.
3817 *
3818 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3819 * not be enough - we need to distinguish between log tree extents vs
3820 * non-log tree extents, and the next filemap_fdatawait_range() call
3821 * will catch and clear such errors in the mapping - and that call might
3822 * be from a log sync and not from a transaction commit. Also, checking
3823 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3824 * not done and would not be reliable - the eb might have been released
3825 * from memory and reading it back again means that flag would not be
3826 * set (since it's a runtime flag, not persisted on disk).
3827 *
3828 * Using the flags below in the btree inode also makes us achieve the
3829 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3830 * writeback for all dirty pages and before filemap_fdatawait_range()
3831 * is called, the writeback for all dirty pages had already finished
3832 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3833 * filemap_fdatawait_range() would return success, as it could not know
3834 * that writeback errors happened (the pages were no longer tagged for
3835 * writeback).
3836 */
3837 switch (eb->log_index) {
3838 case -1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003839 set_bit(BTRFS_FS_BTREE_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003840 break;
3841 case 0:
Josef Bacikafcdd122016-09-02 15:40:02 -04003842 set_bit(BTRFS_FS_LOG1_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003843 break;
3844 case 1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003845 set_bit(BTRFS_FS_LOG2_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003846 break;
3847 default:
3848 BUG(); /* unexpected, logic error */
3849 }
3850}
3851
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003852static void end_bio_extent_buffer_writepage(struct bio *bio)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003853{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003854 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003855 struct extent_buffer *eb;
Christoph Hellwig2b070cf2019-04-25 09:03:00 +02003856 int done;
Ming Lei6dc4f102019-02-15 19:13:19 +08003857 struct bvec_iter_all iter_all;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003858
David Sterbac09abff2017-07-13 18:10:07 +02003859 ASSERT(!bio_flagged(bio, BIO_CLONED));
Christoph Hellwig2b070cf2019-04-25 09:03:00 +02003860 bio_for_each_segment_all(bvec, bio, iter_all) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003861 struct page *page = bvec->bv_page;
3862
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003863 eb = (struct extent_buffer *)page->private;
3864 BUG_ON(!eb);
3865 done = atomic_dec_and_test(&eb->io_pages);
3866
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02003867 if (bio->bi_status ||
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003868 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003869 ClearPageUptodate(page);
Filipe Manana656f30d2014-09-26 12:25:56 +01003870 set_btree_ioerr(page);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003871 }
3872
3873 end_page_writeback(page);
3874
3875 if (!done)
3876 continue;
3877
3878 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003879 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003880
3881 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003882}
3883
Chris Mason0e378df2014-05-19 20:55:27 -07003884static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003885 struct writeback_control *wbc,
3886 struct extent_page_data *epd)
3887{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003888 u64 offset = eb->start;
Liu Bo851cd172016-09-23 13:44:44 -07003889 u32 nritems;
David Sterbacc5e31a2018-03-01 18:20:27 +01003890 int i, num_pages;
Liu Bo851cd172016-09-23 13:44:44 -07003891 unsigned long start, end;
Liu Boff40adf2017-08-24 18:19:48 -06003892 unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003893 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003894
Filipe Manana656f30d2014-09-26 12:25:56 +01003895 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02003896 num_pages = num_extent_pages(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003897 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003898
Liu Bo851cd172016-09-23 13:44:44 -07003899 /* set btree blocks beyond nritems with 0 to avoid stale content. */
3900 nritems = btrfs_header_nritems(eb);
Liu Bo3eb548e2016-09-14 17:22:57 -07003901 if (btrfs_header_level(eb) > 0) {
Liu Bo3eb548e2016-09-14 17:22:57 -07003902 end = btrfs_node_key_ptr_offset(nritems);
3903
David Sterbab159fa22016-11-08 18:09:03 +01003904 memzero_extent_buffer(eb, end, eb->len - end);
Liu Bo851cd172016-09-23 13:44:44 -07003905 } else {
3906 /*
3907 * leaf:
3908 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
3909 */
3910 start = btrfs_item_nr_offset(nritems);
David Sterba8f881e82019-03-20 11:33:10 +01003911 end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(eb);
David Sterbab159fa22016-11-08 18:09:03 +01003912 memzero_extent_buffer(eb, start, end - start);
Liu Bo3eb548e2016-09-14 17:22:57 -07003913 }
3914
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003915 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003916 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003917
3918 clear_page_dirty_for_io(p);
3919 set_page_writeback(p);
David Sterba0ceb34b2020-02-05 19:09:28 +01003920 ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc,
David Sterbafa17ed02019-10-03 17:29:05 +02003921 p, offset, PAGE_SIZE, 0,
David Sterbac2df8bb2017-02-10 19:29:38 +01003922 &epd->bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003923 end_bio_extent_buffer_writepage,
Liu Bo18fdc672017-09-13 12:18:22 -06003924 0, 0, 0, false);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003925 if (ret) {
Filipe Manana656f30d2014-09-26 12:25:56 +01003926 set_btree_ioerr(p);
Takafumi Kubotafe01aa62017-02-09 17:24:33 +09003927 if (PageWriteback(p))
3928 end_page_writeback(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003929 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3930 end_extent_buffer_writeback(eb);
3931 ret = -EIO;
3932 break;
3933 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003934 offset += PAGE_SIZE;
David Sterba3d4b9492017-02-10 19:33:41 +01003935 update_nr_written(wbc, 1);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003936 unlock_page(p);
3937 }
3938
3939 if (unlikely(ret)) {
3940 for (; i < num_pages; i++) {
Chris Masonbbf65cf2014-10-04 09:56:45 -07003941 struct page *p = eb->pages[i];
Liu Bo81465022014-09-23 22:22:33 +08003942 clear_page_dirty_for_io(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003943 unlock_page(p);
3944 }
3945 }
3946
3947 return ret;
3948}
3949
3950int btree_write_cache_pages(struct address_space *mapping,
3951 struct writeback_control *wbc)
3952{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003953 struct extent_buffer *eb, *prev_eb = NULL;
3954 struct extent_page_data epd = {
3955 .bio = NULL,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003956 .extent_locked = 0,
3957 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3958 };
3959 int ret = 0;
3960 int done = 0;
3961 int nr_to_write_done = 0;
3962 struct pagevec pvec;
3963 int nr_pages;
3964 pgoff_t index;
3965 pgoff_t end; /* Inclusive */
3966 int scanned = 0;
Matthew Wilcox10bbd232017-12-05 17:30:38 -05003967 xa_mark_t tag;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003968
Mel Gorman86679822017-11-15 17:37:52 -08003969 pagevec_init(&pvec);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003970 if (wbc->range_cyclic) {
3971 index = mapping->writeback_index; /* Start from prev offset */
3972 end = -1;
Josef Bacik556755a2020-01-03 10:38:44 -05003973 /*
3974 * Start from the beginning does not need to cycle over the
3975 * range, mark it as scanned.
3976 */
3977 scanned = (index == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003978 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003979 index = wbc->range_start >> PAGE_SHIFT;
3980 end = wbc->range_end >> PAGE_SHIFT;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003981 scanned = 1;
3982 }
3983 if (wbc->sync_mode == WB_SYNC_ALL)
3984 tag = PAGECACHE_TAG_TOWRITE;
3985 else
3986 tag = PAGECACHE_TAG_DIRTY;
3987retry:
3988 if (wbc->sync_mode == WB_SYNC_ALL)
3989 tag_pages_for_writeback(mapping, index, end);
3990 while (!done && !nr_to_write_done && (index <= end) &&
Jan Kara4006f432017-11-15 17:34:37 -08003991 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
Jan Kara67fd7072017-11-15 17:35:19 -08003992 tag))) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003993 unsigned i;
3994
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003995 for (i = 0; i < nr_pages; i++) {
3996 struct page *page = pvec.pages[i];
3997
3998 if (!PagePrivate(page))
3999 continue;
4000
Josef Bacikb5bae262012-09-14 13:43:01 -04004001 spin_lock(&mapping->private_lock);
4002 if (!PagePrivate(page)) {
4003 spin_unlock(&mapping->private_lock);
4004 continue;
4005 }
4006
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004007 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04004008
4009 /*
4010 * Shouldn't happen and normally this would be a BUG_ON
4011 * but no sense in crashing the users box for something
4012 * we can survive anyway.
4013 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05304014 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04004015 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004016 continue;
4017 }
4018
Josef Bacikb5bae262012-09-14 13:43:01 -04004019 if (eb == prev_eb) {
4020 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004021 continue;
4022 }
4023
Josef Bacikb5bae262012-09-14 13:43:01 -04004024 ret = atomic_inc_not_zero(&eb->refs);
4025 spin_unlock(&mapping->private_lock);
4026 if (!ret)
4027 continue;
4028
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004029 prev_eb = eb;
David Sterba9df76fb2019-03-20 11:21:41 +01004030 ret = lock_extent_buffer_for_io(eb, &epd);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004031 if (!ret) {
4032 free_extent_buffer(eb);
4033 continue;
Filipe Manana0607eb1d2019-09-11 17:42:28 +01004034 } else if (ret < 0) {
4035 done = 1;
4036 free_extent_buffer(eb);
4037 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004038 }
4039
David Sterba0ab02062019-03-20 11:27:57 +01004040 ret = write_one_eb(eb, wbc, &epd);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004041 if (ret) {
4042 done = 1;
4043 free_extent_buffer(eb);
4044 break;
4045 }
4046 free_extent_buffer(eb);
4047
4048 /*
4049 * the filesystem may choose to bump up nr_to_write.
4050 * We have to make sure to honor the new nr_to_write
4051 * at any time
4052 */
4053 nr_to_write_done = wbc->nr_to_write <= 0;
4054 }
4055 pagevec_release(&pvec);
4056 cond_resched();
4057 }
4058 if (!scanned && !done) {
4059 /*
4060 * We hit the last page and there is more work to be done: wrap
4061 * back to the start of the file
4062 */
4063 scanned = 1;
4064 index = 0;
4065 goto retry;
4066 }
Qu Wenruo2b952ee2019-03-20 14:27:43 +08004067 ASSERT(ret <= 0);
4068 if (ret < 0) {
4069 end_write_bio(&epd, ret);
4070 return ret;
4071 }
4072 ret = flush_write_bio(&epd);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004073 return ret;
4074}
4075
Chris Masond1310b22008-01-24 16:13:08 -05004076/**
Chris Mason4bef0842008-09-08 11:18:08 -04004077 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
Chris Masond1310b22008-01-24 16:13:08 -05004078 * @mapping: address space structure to write
4079 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
David Sterba935db852017-06-23 04:30:28 +02004080 * @data: data passed to __extent_writepage function
Chris Masond1310b22008-01-24 16:13:08 -05004081 *
4082 * If a page is already under I/O, write_cache_pages() skips it, even
4083 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
4084 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
4085 * and msync() need to guarantee that all the data which was dirty at the time
4086 * the call was made get new I/O started against them. If wbc->sync_mode is
4087 * WB_SYNC_ALL then we were called for data integrity and we must wait for
4088 * existing IO to complete.
4089 */
David Sterba4242b642017-02-10 19:38:24 +01004090static int extent_write_cache_pages(struct address_space *mapping,
Chris Mason4bef0842008-09-08 11:18:08 -04004091 struct writeback_control *wbc,
David Sterbaaab6e9e2017-11-30 18:00:02 +01004092 struct extent_page_data *epd)
Chris Masond1310b22008-01-24 16:13:08 -05004093{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004094 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05004095 int ret = 0;
4096 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004097 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004098 struct pagevec pvec;
4099 int nr_pages;
4100 pgoff_t index;
4101 pgoff_t end; /* Inclusive */
Liu Boa91326672016-03-07 16:56:21 -08004102 pgoff_t done_index;
4103 int range_whole = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004104 int scanned = 0;
Matthew Wilcox10bbd232017-12-05 17:30:38 -05004105 xa_mark_t tag;
Chris Masond1310b22008-01-24 16:13:08 -05004106
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004107 /*
4108 * We have to hold onto the inode so that ordered extents can do their
4109 * work when the IO finishes. The alternative to this is failing to add
4110 * an ordered extent if the igrab() fails there and that is a huge pain
4111 * to deal with, so instead just hold onto the inode throughout the
4112 * writepages operation. If it fails here we are freeing up the inode
4113 * anyway and we'd rather not waste our time writing out stuff that is
4114 * going to be truncated anyway.
4115 */
4116 if (!igrab(inode))
4117 return 0;
4118
Mel Gorman86679822017-11-15 17:37:52 -08004119 pagevec_init(&pvec);
Chris Masond1310b22008-01-24 16:13:08 -05004120 if (wbc->range_cyclic) {
4121 index = mapping->writeback_index; /* Start from prev offset */
4122 end = -1;
Josef Bacik556755a2020-01-03 10:38:44 -05004123 /*
4124 * Start from the beginning does not need to cycle over the
4125 * range, mark it as scanned.
4126 */
4127 scanned = (index == 0);
Chris Masond1310b22008-01-24 16:13:08 -05004128 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004129 index = wbc->range_start >> PAGE_SHIFT;
4130 end = wbc->range_end >> PAGE_SHIFT;
Liu Boa91326672016-03-07 16:56:21 -08004131 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
4132 range_whole = 1;
Chris Masond1310b22008-01-24 16:13:08 -05004133 scanned = 1;
4134 }
Ethan Lien3cd24c62018-11-01 14:49:03 +08004135
4136 /*
4137 * We do the tagged writepage as long as the snapshot flush bit is set
4138 * and we are the first one who do the filemap_flush() on this inode.
4139 *
4140 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
4141 * not race in and drop the bit.
4142 */
4143 if (range_whole && wbc->nr_to_write == LONG_MAX &&
4144 test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
4145 &BTRFS_I(inode)->runtime_flags))
4146 wbc->tagged_writepages = 1;
4147
4148 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Josef Bacikf7aaa062011-07-15 21:26:38 +00004149 tag = PAGECACHE_TAG_TOWRITE;
4150 else
4151 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05004152retry:
Ethan Lien3cd24c62018-11-01 14:49:03 +08004153 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Josef Bacikf7aaa062011-07-15 21:26:38 +00004154 tag_pages_for_writeback(mapping, index, end);
Liu Boa91326672016-03-07 16:56:21 -08004155 done_index = index;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004156 while (!done && !nr_to_write_done && (index <= end) &&
Jan Kara67fd7072017-11-15 17:35:19 -08004157 (nr_pages = pagevec_lookup_range_tag(&pvec, mapping,
4158 &index, end, tag))) {
Chris Masond1310b22008-01-24 16:13:08 -05004159 unsigned i;
4160
Chris Masond1310b22008-01-24 16:13:08 -05004161 for (i = 0; i < nr_pages; i++) {
4162 struct page *page = pvec.pages[i];
4163
Tejun Heof7bddf12019-10-03 07:27:13 -07004164 done_index = page->index + 1;
Chris Masond1310b22008-01-24 16:13:08 -05004165 /*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07004166 * At this point we hold neither the i_pages lock nor
4167 * the page lock: the page may be truncated or
4168 * invalidated (changing page->mapping to NULL),
4169 * or even swizzled back from swapper_space to
4170 * tmpfs file mapping
Chris Masond1310b22008-01-24 16:13:08 -05004171 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05004172 if (!trylock_page(page)) {
Qu Wenruof4340622019-03-20 14:27:41 +08004173 ret = flush_write_bio(epd);
4174 BUG_ON(ret < 0);
Josef Bacikc8f2f242013-02-11 11:33:00 -05004175 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04004176 }
Chris Masond1310b22008-01-24 16:13:08 -05004177
4178 if (unlikely(page->mapping != mapping)) {
4179 unlock_page(page);
4180 continue;
4181 }
4182
Chris Masond2c3f4f2008-11-19 12:44:22 -05004183 if (wbc->sync_mode != WB_SYNC_NONE) {
Qu Wenruof4340622019-03-20 14:27:41 +08004184 if (PageWriteback(page)) {
4185 ret = flush_write_bio(epd);
4186 BUG_ON(ret < 0);
4187 }
Chris Masond1310b22008-01-24 16:13:08 -05004188 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004189 }
Chris Masond1310b22008-01-24 16:13:08 -05004190
4191 if (PageWriteback(page) ||
4192 !clear_page_dirty_for_io(page)) {
4193 unlock_page(page);
4194 continue;
4195 }
4196
David Sterbaaab6e9e2017-11-30 18:00:02 +01004197 ret = __extent_writepage(page, wbc, epd);
Liu Boa91326672016-03-07 16:56:21 -08004198 if (ret < 0) {
Liu Boa91326672016-03-07 16:56:21 -08004199 done = 1;
4200 break;
4201 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004202
4203 /*
4204 * the filesystem may choose to bump up nr_to_write.
4205 * We have to make sure to honor the new nr_to_write
4206 * at any time
4207 */
4208 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05004209 }
4210 pagevec_release(&pvec);
4211 cond_resched();
4212 }
Liu Bo894b36e2016-03-07 16:56:22 -08004213 if (!scanned && !done) {
Chris Masond1310b22008-01-24 16:13:08 -05004214 /*
4215 * We hit the last page and there is more work to be done: wrap
4216 * back to the start of the file
4217 */
4218 scanned = 1;
4219 index = 0;
Josef Bacik42ffb0b2020-01-23 15:33:02 -05004220
4221 /*
4222 * If we're looping we could run into a page that is locked by a
4223 * writer and that writer could be waiting on writeback for a
4224 * page in our current bio, and thus deadlock, so flush the
4225 * write bio here.
4226 */
4227 ret = flush_write_bio(epd);
4228 if (!ret)
4229 goto retry;
Chris Masond1310b22008-01-24 16:13:08 -05004230 }
Liu Boa91326672016-03-07 16:56:21 -08004231
4232 if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
4233 mapping->writeback_index = done_index;
4234
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004235 btrfs_add_delayed_iput(inode);
Liu Bo894b36e2016-03-07 16:56:22 -08004236 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05004237}
Chris Masond1310b22008-01-24 16:13:08 -05004238
Nikolay Borisov0a9b0e52017-12-08 15:55:59 +02004239int extent_write_full_page(struct page *page, struct writeback_control *wbc)
Chris Masond1310b22008-01-24 16:13:08 -05004240{
4241 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004242 struct extent_page_data epd = {
4243 .bio = NULL,
Chris Mason771ed682008-11-06 22:02:51 -05004244 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004245 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05004246 };
Chris Masond1310b22008-01-24 16:13:08 -05004247
Chris Masond1310b22008-01-24 16:13:08 -05004248 ret = __extent_writepage(page, wbc, &epd);
Qu Wenruo30659762019-03-20 14:27:42 +08004249 ASSERT(ret <= 0);
4250 if (ret < 0) {
4251 end_write_bio(&epd, ret);
4252 return ret;
4253 }
Chris Masond1310b22008-01-24 16:13:08 -05004254
Qu Wenruo30659762019-03-20 14:27:42 +08004255 ret = flush_write_bio(&epd);
4256 ASSERT(ret <= 0);
Chris Masond1310b22008-01-24 16:13:08 -05004257 return ret;
4258}
Chris Masond1310b22008-01-24 16:13:08 -05004259
Nikolay Borisov5e3ee232017-12-08 15:55:58 +02004260int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
Chris Mason771ed682008-11-06 22:02:51 -05004261 int mode)
4262{
4263 int ret = 0;
4264 struct address_space *mapping = inode->i_mapping;
4265 struct page *page;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004266 unsigned long nr_pages = (end - start + PAGE_SIZE) >>
4267 PAGE_SHIFT;
Chris Mason771ed682008-11-06 22:02:51 -05004268
4269 struct extent_page_data epd = {
4270 .bio = NULL,
Chris Mason771ed682008-11-06 22:02:51 -05004271 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04004272 .sync_io = mode == WB_SYNC_ALL,
Chris Mason771ed682008-11-06 22:02:51 -05004273 };
4274 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05004275 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05004276 .nr_to_write = nr_pages * 2,
4277 .range_start = start,
4278 .range_end = end + 1,
Chris Masonec39f762019-07-10 12:28:17 -07004279 /* We're called from an async helper function */
4280 .punt_to_cgroup = 1,
4281 .no_cgroup_owner = 1,
Chris Mason771ed682008-11-06 22:02:51 -05004282 };
4283
Chris Masondbb70be2019-07-10 12:28:18 -07004284 wbc_attach_fdatawrite_inode(&wbc_writepages, inode);
Chris Masond3977122009-01-05 21:25:51 -05004285 while (start <= end) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004286 page = find_get_page(mapping, start >> PAGE_SHIFT);
Chris Mason771ed682008-11-06 22:02:51 -05004287 if (clear_page_dirty_for_io(page))
4288 ret = __extent_writepage(page, &wbc_writepages, &epd);
4289 else {
Nikolay Borisov7087a9d2018-11-01 14:09:48 +02004290 btrfs_writepage_endio_finish_ordered(page, start,
Nikolay Borisovc6297322018-11-08 10:18:08 +02004291 start + PAGE_SIZE - 1, 1);
Chris Mason771ed682008-11-06 22:02:51 -05004292 unlock_page(page);
4293 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004294 put_page(page);
4295 start += PAGE_SIZE;
Chris Mason771ed682008-11-06 22:02:51 -05004296 }
4297
Qu Wenruo02c6db42019-03-20 14:27:45 +08004298 ASSERT(ret <= 0);
Chris Masondbb70be2019-07-10 12:28:18 -07004299 if (ret == 0)
4300 ret = flush_write_bio(&epd);
4301 else
Qu Wenruo02c6db42019-03-20 14:27:45 +08004302 end_write_bio(&epd, ret);
Chris Masondbb70be2019-07-10 12:28:18 -07004303
4304 wbc_detach_inode(&wbc_writepages);
Chris Mason771ed682008-11-06 22:02:51 -05004305 return ret;
4306}
Chris Masond1310b22008-01-24 16:13:08 -05004307
Nikolay Borisov8ae225a2018-04-19 10:46:38 +03004308int extent_writepages(struct address_space *mapping,
Chris Masond1310b22008-01-24 16:13:08 -05004309 struct writeback_control *wbc)
4310{
4311 int ret = 0;
4312 struct extent_page_data epd = {
4313 .bio = NULL,
Chris Mason771ed682008-11-06 22:02:51 -05004314 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004315 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05004316 };
4317
David Sterba935db852017-06-23 04:30:28 +02004318 ret = extent_write_cache_pages(mapping, wbc, &epd);
Qu Wenruoa2a72fb2019-03-20 14:27:48 +08004319 ASSERT(ret <= 0);
4320 if (ret < 0) {
4321 end_write_bio(&epd, ret);
4322 return ret;
4323 }
4324 ret = flush_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004325 return ret;
4326}
Chris Masond1310b22008-01-24 16:13:08 -05004327
Nikolay Borisov2a3ff0a2018-04-19 10:46:36 +03004328int extent_readpages(struct address_space *mapping, struct list_head *pages,
4329 unsigned nr_pages)
Chris Masond1310b22008-01-24 16:13:08 -05004330{
4331 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04004332 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06004333 struct page *pagepool[16];
Miao Xie125bac012013-07-25 19:22:37 +08004334 struct extent_map *em_cached = NULL;
Nikolay Borisov2a3ff0a2018-04-19 10:46:36 +03004335 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
Liu Bo67c96842012-07-20 21:43:09 -06004336 int nr = 0;
Filipe Manana808f80b2015-09-28 09:56:26 +01004337 u64 prev_em_start = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05004338
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004339 while (!list_empty(pages)) {
Nikolay Borisove65ef212019-03-11 09:55:38 +02004340 u64 contig_end = 0;
4341
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004342 for (nr = 0; nr < ARRAY_SIZE(pagepool) && !list_empty(pages);) {
Nikolay Borisovf86196e2019-01-03 15:29:02 -08004343 struct page *page = lru_to_page(pages);
Chris Masond1310b22008-01-24 16:13:08 -05004344
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004345 prefetchw(&page->flags);
4346 list_del(&page->lru);
4347 if (add_to_page_cache_lru(page, mapping, page->index,
4348 readahead_gfp_mask(mapping))) {
4349 put_page(page);
Nikolay Borisove65ef212019-03-11 09:55:38 +02004350 break;
Nikolay Borisov61ed3a12018-11-29 18:41:31 +02004351 }
4352
4353 pagepool[nr++] = page;
Nikolay Borisove65ef212019-03-11 09:55:38 +02004354 contig_end = page_offset(page) + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05004355 }
Liu Bo67c96842012-07-20 21:43:09 -06004356
Nikolay Borisove65ef212019-03-11 09:55:38 +02004357 if (nr) {
4358 u64 contig_start = page_offset(pagepool[0]);
4359
4360 ASSERT(contig_start + nr * PAGE_SIZE - 1 == contig_end);
4361
4362 contiguous_readpages(tree, pagepool, nr, contig_start,
4363 contig_end, &em_cached, &bio, &bio_flags,
4364 &prev_em_start);
4365 }
Chris Masond1310b22008-01-24 16:13:08 -05004366 }
Liu Bo67c96842012-07-20 21:43:09 -06004367
Miao Xie125bac012013-07-25 19:22:37 +08004368 if (em_cached)
4369 free_extent_map(em_cached);
4370
Chris Masond1310b22008-01-24 16:13:08 -05004371 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05004372 return submit_one_bio(bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004373 return 0;
4374}
Chris Masond1310b22008-01-24 16:13:08 -05004375
4376/*
4377 * basic invalidatepage code, this waits on any locked or writeback
4378 * ranges corresponding to the page, and then deletes any extent state
4379 * records from the tree
4380 */
4381int extent_invalidatepage(struct extent_io_tree *tree,
4382 struct page *page, unsigned long offset)
4383{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004384 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004385 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004386 u64 end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05004387 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4388
Qu Wenruofda28322013-02-26 08:10:22 +00004389 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004390 if (start > end)
4391 return 0;
4392
David Sterbaff13db42015-12-03 14:30:40 +01004393 lock_extent_bits(tree, start, end, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004394 wait_on_page_writeback(page);
Omar Sandovale1821632019-08-15 14:04:04 -07004395 clear_extent_bit(tree, start, end, EXTENT_LOCKED | EXTENT_DELALLOC |
4396 EXTENT_DO_ACCOUNTING, 1, 1, &cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05004397 return 0;
4398}
Chris Masond1310b22008-01-24 16:13:08 -05004399
4400/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004401 * a helper for releasepage, this tests for areas of the page that
4402 * are locked or under IO and drops the related state bits if it is safe
4403 * to drop the page.
4404 */
Nikolay Borisov29c68b2d2018-04-19 10:46:35 +03004405static int try_release_extent_state(struct extent_io_tree *tree,
Eric Sandeen48a3b632013-04-25 20:41:01 +00004406 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004407{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004408 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004409 u64 end = start + PAGE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004410 int ret = 1;
4411
Nikolay Borisov88826792019-03-14 15:28:31 +02004412 if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) {
Chris Mason7b13b7b2008-04-18 10:29:50 -04004413 ret = 0;
Nikolay Borisov88826792019-03-14 15:28:31 +02004414 } else {
Chris Mason11ef1602009-09-23 20:28:46 -04004415 /*
4416 * at this point we can safely clear everything except the
4417 * locked bit and the nodatasum bit
4418 */
David Sterba66b0c882017-10-31 16:30:47 +01004419 ret = __clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004420 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
David Sterba66b0c882017-10-31 16:30:47 +01004421 0, 0, NULL, mask, NULL);
Chris Masone3f24cc2011-02-14 12:52:08 -05004422
4423 /* if clear_extent_bit failed for enomem reasons,
4424 * we can't allow the release to continue.
4425 */
4426 if (ret < 0)
4427 ret = 0;
4428 else
4429 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004430 }
4431 return ret;
4432}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004433
4434/*
Chris Masond1310b22008-01-24 16:13:08 -05004435 * a helper for releasepage. As long as there are no locked extents
4436 * in the range corresponding to the page, both state records and extent
4437 * map records are removed
4438 */
Nikolay Borisov477a30b2018-04-19 10:46:34 +03004439int try_release_extent_mapping(struct page *page, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004440{
4441 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004442 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004443 u64 end = start + PAGE_SIZE - 1;
Filipe Mananabd3599a2018-07-12 01:36:43 +01004444 struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
4445 struct extent_io_tree *tree = &btrfs_inode->io_tree;
4446 struct extent_map_tree *map = &btrfs_inode->extent_tree;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004447
Mel Gormand0164ad2015-11-06 16:28:21 -08004448 if (gfpflags_allow_blocking(mask) &&
Byongho Leeee221842015-12-15 01:42:10 +09004449 page->mapping->host->i_size > SZ_16M) {
Yan39b56372008-02-15 10:40:50 -05004450 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004451 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004452 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004453 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004454 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004455 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004456 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004457 break;
4458 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004459 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4460 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004461 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004462 free_extent_map(em);
4463 break;
4464 }
4465 if (!test_range_bit(tree, em->start,
4466 extent_map_end(em) - 1,
Nikolay Borisov4e586ca2019-03-14 15:28:30 +02004467 EXTENT_LOCKED, 0, NULL)) {
Filipe Mananabd3599a2018-07-12 01:36:43 +01004468 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4469 &btrfs_inode->runtime_flags);
Chris Mason70dec802008-01-29 09:59:12 -05004470 remove_extent_mapping(map, em);
4471 /* once for the rb tree */
4472 free_extent_map(em);
4473 }
4474 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004475 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004476
4477 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004478 free_extent_map(em);
4479 }
Chris Masond1310b22008-01-24 16:13:08 -05004480 }
Nikolay Borisov29c68b2d2018-04-19 10:46:35 +03004481 return try_release_extent_state(tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004482}
Chris Masond1310b22008-01-24 16:13:08 -05004483
Chris Masonec29ed52011-02-23 16:23:20 -05004484/*
4485 * helper function for fiemap, which doesn't want to see any holes.
4486 * This maps until we find something past 'last'
4487 */
4488static struct extent_map *get_extent_skip_holes(struct inode *inode,
David Sterbae3350e12017-06-23 04:09:57 +02004489 u64 offset, u64 last)
Chris Masonec29ed52011-02-23 16:23:20 -05004490{
Jeff Mahoneyda170662016-06-15 09:22:56 -04004491 u64 sectorsize = btrfs_inode_sectorsize(inode);
Chris Masonec29ed52011-02-23 16:23:20 -05004492 struct extent_map *em;
4493 u64 len;
4494
4495 if (offset >= last)
4496 return NULL;
4497
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304498 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004499 len = last - offset;
4500 if (len == 0)
4501 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004502 len = ALIGN(len, sectorsize);
Nikolay Borisov4ab47a82018-12-12 09:42:32 +02004503 em = btrfs_get_extent_fiemap(BTRFS_I(inode), offset, len);
David Sterbac7040052011-04-19 18:00:01 +02004504 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004505 return em;
4506
4507 /* if this isn't a hole return it */
Nikolay Borisov4a2d25c2017-11-23 10:51:43 +02004508 if (em->block_start != EXTENT_MAP_HOLE)
Chris Masonec29ed52011-02-23 16:23:20 -05004509 return em;
Chris Masonec29ed52011-02-23 16:23:20 -05004510
4511 /* this is a hole, advance to the next extent */
4512 offset = extent_map_end(em);
4513 free_extent_map(em);
4514 if (offset >= last)
4515 break;
4516 }
4517 return NULL;
4518}
4519
Qu Wenruo47518322017-04-07 10:43:15 +08004520/*
4521 * To cache previous fiemap extent
4522 *
4523 * Will be used for merging fiemap extent
4524 */
4525struct fiemap_cache {
4526 u64 offset;
4527 u64 phys;
4528 u64 len;
4529 u32 flags;
4530 bool cached;
4531};
4532
4533/*
4534 * Helper to submit fiemap extent.
4535 *
4536 * Will try to merge current fiemap extent specified by @offset, @phys,
4537 * @len and @flags with cached one.
4538 * And only when we fails to merge, cached one will be submitted as
4539 * fiemap extent.
4540 *
4541 * Return value is the same as fiemap_fill_next_extent().
4542 */
4543static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
4544 struct fiemap_cache *cache,
4545 u64 offset, u64 phys, u64 len, u32 flags)
4546{
4547 int ret = 0;
4548
4549 if (!cache->cached)
4550 goto assign;
4551
4552 /*
4553 * Sanity check, extent_fiemap() should have ensured that new
Andrea Gelmini52042d82018-11-28 12:05:13 +01004554 * fiemap extent won't overlap with cached one.
Qu Wenruo47518322017-04-07 10:43:15 +08004555 * Not recoverable.
4556 *
4557 * NOTE: Physical address can overlap, due to compression
4558 */
4559 if (cache->offset + cache->len > offset) {
4560 WARN_ON(1);
4561 return -EINVAL;
4562 }
4563
4564 /*
4565 * Only merges fiemap extents if
4566 * 1) Their logical addresses are continuous
4567 *
4568 * 2) Their physical addresses are continuous
4569 * So truly compressed (physical size smaller than logical size)
4570 * extents won't get merged with each other
4571 *
4572 * 3) Share same flags except FIEMAP_EXTENT_LAST
4573 * So regular extent won't get merged with prealloc extent
4574 */
4575 if (cache->offset + cache->len == offset &&
4576 cache->phys + cache->len == phys &&
4577 (cache->flags & ~FIEMAP_EXTENT_LAST) ==
4578 (flags & ~FIEMAP_EXTENT_LAST)) {
4579 cache->len += len;
4580 cache->flags |= flags;
4581 goto try_submit_last;
4582 }
4583
4584 /* Not mergeable, need to submit cached one */
4585 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4586 cache->len, cache->flags);
4587 cache->cached = false;
4588 if (ret)
4589 return ret;
4590assign:
4591 cache->cached = true;
4592 cache->offset = offset;
4593 cache->phys = phys;
4594 cache->len = len;
4595 cache->flags = flags;
4596try_submit_last:
4597 if (cache->flags & FIEMAP_EXTENT_LAST) {
4598 ret = fiemap_fill_next_extent(fieinfo, cache->offset,
4599 cache->phys, cache->len, cache->flags);
4600 cache->cached = false;
4601 }
4602 return ret;
4603}
4604
4605/*
Qu Wenruo848c23b2017-06-22 10:01:21 +08004606 * Emit last fiemap cache
Qu Wenruo47518322017-04-07 10:43:15 +08004607 *
Qu Wenruo848c23b2017-06-22 10:01:21 +08004608 * The last fiemap cache may still be cached in the following case:
4609 * 0 4k 8k
4610 * |<- Fiemap range ->|
4611 * |<------------ First extent ----------->|
4612 *
4613 * In this case, the first extent range will be cached but not emitted.
4614 * So we must emit it before ending extent_fiemap().
Qu Wenruo47518322017-04-07 10:43:15 +08004615 */
David Sterba5c5aff92019-03-20 11:29:46 +01004616static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
Qu Wenruo848c23b2017-06-22 10:01:21 +08004617 struct fiemap_cache *cache)
Qu Wenruo47518322017-04-07 10:43:15 +08004618{
4619 int ret;
4620
4621 if (!cache->cached)
4622 return 0;
4623
Qu Wenruo47518322017-04-07 10:43:15 +08004624 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4625 cache->len, cache->flags);
4626 cache->cached = false;
4627 if (ret > 0)
4628 ret = 0;
4629 return ret;
4630}
4631
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004632int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
David Sterba2135fb92017-06-23 04:09:57 +02004633 __u64 start, __u64 len)
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004634{
Josef Bacik975f84f2010-11-23 19:36:57 +00004635 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004636 u64 off = start;
4637 u64 max = start + len;
4638 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004639 u32 found_type;
4640 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004641 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004642 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004643 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004644 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004645 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004646 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004647 struct btrfs_path *path;
Josef Bacikdc046b12014-09-10 16:20:45 -04004648 struct btrfs_root *root = BTRFS_I(inode)->root;
Qu Wenruo47518322017-04-07 10:43:15 +08004649 struct fiemap_cache cache = { 0 };
David Sterba5911c8f2019-05-15 15:31:04 +02004650 struct ulist *roots;
4651 struct ulist *tmp_ulist;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004652 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004653 u64 em_start = 0;
4654 u64 em_len = 0;
4655 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004656
4657 if (len == 0)
4658 return -EINVAL;
4659
Josef Bacik975f84f2010-11-23 19:36:57 +00004660 path = btrfs_alloc_path();
4661 if (!path)
4662 return -ENOMEM;
4663 path->leave_spinning = 1;
4664
David Sterba5911c8f2019-05-15 15:31:04 +02004665 roots = ulist_alloc(GFP_KERNEL);
4666 tmp_ulist = ulist_alloc(GFP_KERNEL);
4667 if (!roots || !tmp_ulist) {
4668 ret = -ENOMEM;
4669 goto out_free_ulist;
4670 }
4671
Jeff Mahoneyda170662016-06-15 09:22:56 -04004672 start = round_down(start, btrfs_inode_sectorsize(inode));
4673 len = round_up(max, btrfs_inode_sectorsize(inode)) - start;
Josef Bacik4d479cf2011-11-17 11:34:31 -05004674
Chris Masonec29ed52011-02-23 16:23:20 -05004675 /*
4676 * lookup the last file extent. We're not using i_size here
4677 * because there might be preallocation past i_size
4678 */
David Sterbaf85b7372017-01-20 14:54:07 +01004679 ret = btrfs_lookup_file_extent(NULL, root, path,
4680 btrfs_ino(BTRFS_I(inode)), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004681 if (ret < 0) {
David Sterba5911c8f2019-05-15 15:31:04 +02004682 goto out_free_ulist;
Liu Bo2d324f52016-05-17 17:21:48 -07004683 } else {
4684 WARN_ON(!ret);
4685 if (ret == 1)
4686 ret = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004687 }
Liu Bo2d324f52016-05-17 17:21:48 -07004688
Josef Bacik975f84f2010-11-23 19:36:57 +00004689 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004690 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
David Sterba962a2982014-06-04 18:41:45 +02004691 found_type = found_key.type;
Josef Bacik975f84f2010-11-23 19:36:57 +00004692
Chris Masonec29ed52011-02-23 16:23:20 -05004693 /* No extents, but there might be delalloc bits */
Nikolay Borisov4a0cc7c2017-01-10 20:35:31 +02004694 if (found_key.objectid != btrfs_ino(BTRFS_I(inode)) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004695 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004696 /* have to trust i_size as the end */
4697 last = (u64)-1;
4698 last_for_get_extent = isize;
4699 } else {
4700 /*
4701 * remember the start of the last extent. There are a
4702 * bunch of different factors that go into the length of the
4703 * extent, so its much less complex to remember where it started
4704 */
4705 last = found_key.offset;
4706 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004707 }
Liu Bofe09e162013-09-22 12:54:23 +08004708 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004709
Chris Masonec29ed52011-02-23 16:23:20 -05004710 /*
4711 * we might have some extents allocated but more delalloc past those
4712 * extents. so, we trust isize unless the start of the last extent is
4713 * beyond isize
4714 */
4715 if (last < isize) {
4716 last = (u64)-1;
4717 last_for_get_extent = isize;
4718 }
4719
David Sterbaff13db42015-12-03 14:30:40 +01004720 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004721 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004722
David Sterbae3350e12017-06-23 04:09:57 +02004723 em = get_extent_skip_holes(inode, start, last_for_get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004724 if (!em)
4725 goto out;
4726 if (IS_ERR(em)) {
4727 ret = PTR_ERR(em);
4728 goto out;
4729 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004730
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004731 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004732 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004733
Chris Masonea8efc72011-03-08 11:54:40 -05004734 /* break if the extent we found is outside the range */
4735 if (em->start >= max || extent_map_end(em) < off)
4736 break;
4737
4738 /*
4739 * get_extent may return an extent that starts before our
4740 * requested range. We have to make sure the ranges
4741 * we return to fiemap always move forward and don't
4742 * overlap, so adjust the offsets here
4743 */
4744 em_start = max(em->start, off);
4745
4746 /*
4747 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004748 * for adjusting the disk offset below. Only do this if the
4749 * extent isn't compressed since our in ram offset may be past
4750 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004751 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004752 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4753 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004754 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004755 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004756 flags = 0;
Filipe Mananaf0986312018-06-20 10:02:30 +01004757 if (em->block_start < EXTENT_MAP_LAST_BYTE)
4758 disko = em->block_start + offset_in_extent;
4759 else
4760 disko = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004761
Chris Masonea8efc72011-03-08 11:54:40 -05004762 /*
4763 * bump off for our next call to get_extent
4764 */
4765 off = extent_map_end(em);
4766 if (off >= max)
4767 end = 1;
4768
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004769 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004770 end = 1;
4771 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004772 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004773 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4774 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004775 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004776 flags |= (FIEMAP_EXTENT_DELALLOC |
4777 FIEMAP_EXTENT_UNKNOWN);
Josef Bacikdc046b12014-09-10 16:20:45 -04004778 } else if (fieinfo->fi_extents_max) {
4779 u64 bytenr = em->block_start -
4780 (em->start - em->orig_start);
Liu Bofe09e162013-09-22 12:54:23 +08004781
Liu Bofe09e162013-09-22 12:54:23 +08004782 /*
4783 * As btrfs supports shared space, this information
4784 * can be exported to userspace tools via
Josef Bacikdc046b12014-09-10 16:20:45 -04004785 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4786 * then we're just getting a count and we can skip the
4787 * lookup stuff.
Liu Bofe09e162013-09-22 12:54:23 +08004788 */
Edmund Nadolskibb739cf2017-06-28 21:56:58 -06004789 ret = btrfs_check_shared(root,
4790 btrfs_ino(BTRFS_I(inode)),
David Sterba5911c8f2019-05-15 15:31:04 +02004791 bytenr, roots, tmp_ulist);
Josef Bacikdc046b12014-09-10 16:20:45 -04004792 if (ret < 0)
Liu Bofe09e162013-09-22 12:54:23 +08004793 goto out_free;
Josef Bacikdc046b12014-09-10 16:20:45 -04004794 if (ret)
Liu Bofe09e162013-09-22 12:54:23 +08004795 flags |= FIEMAP_EXTENT_SHARED;
Josef Bacikdc046b12014-09-10 16:20:45 -04004796 ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004797 }
4798 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4799 flags |= FIEMAP_EXTENT_ENCODED;
Josef Bacik0d2b2372015-05-19 10:44:04 -04004800 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4801 flags |= FIEMAP_EXTENT_UNWRITTEN;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004802
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004803 free_extent_map(em);
4804 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004805 if ((em_start >= last) || em_len == (u64)-1 ||
4806 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004807 flags |= FIEMAP_EXTENT_LAST;
4808 end = 1;
4809 }
4810
Chris Masonec29ed52011-02-23 16:23:20 -05004811 /* now scan forward to see if this is really the last extent. */
David Sterbae3350e12017-06-23 04:09:57 +02004812 em = get_extent_skip_holes(inode, off, last_for_get_extent);
Chris Masonec29ed52011-02-23 16:23:20 -05004813 if (IS_ERR(em)) {
4814 ret = PTR_ERR(em);
4815 goto out;
4816 }
4817 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004818 flags |= FIEMAP_EXTENT_LAST;
4819 end = 1;
4820 }
Qu Wenruo47518322017-04-07 10:43:15 +08004821 ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko,
4822 em_len, flags);
Chengyu Song26e726a2015-03-24 18:12:56 -04004823 if (ret) {
4824 if (ret == 1)
4825 ret = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004826 goto out_free;
Chengyu Song26e726a2015-03-24 18:12:56 -04004827 }
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004828 }
4829out_free:
Qu Wenruo47518322017-04-07 10:43:15 +08004830 if (!ret)
David Sterba5c5aff92019-03-20 11:29:46 +01004831 ret = emit_last_fiemap_cache(fieinfo, &cache);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004832 free_extent_map(em);
4833out:
Liu Boa52f4cd2013-05-01 16:23:41 +00004834 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
David Sterbae43bbe52017-12-12 21:43:52 +01004835 &cached_state);
David Sterba5911c8f2019-05-15 15:31:04 +02004836
4837out_free_ulist:
Colin Ian Kinge02d48e2019-07-05 08:26:24 +01004838 btrfs_free_path(path);
David Sterba5911c8f2019-05-15 15:31:04 +02004839 ulist_free(roots);
4840 ulist_free(tmp_ulist);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004841 return ret;
4842}
4843
Chris Mason727011e2010-08-06 13:21:20 -04004844static void __free_extent_buffer(struct extent_buffer *eb)
4845{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004846 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004847 kmem_cache_free(extent_buffer_cache, eb);
4848}
4849
Josef Bacika26e8c92014-03-28 17:07:27 -04004850int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004851{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004852 return (atomic_read(&eb->io_pages) ||
4853 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4854 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004855}
4856
Miao Xie897ca6e92010-10-26 20:57:29 -04004857/*
David Sterba55ac0132018-07-19 17:24:32 +02004858 * Release all pages attached to the extent buffer.
Miao Xie897ca6e92010-10-26 20:57:29 -04004859 */
David Sterba55ac0132018-07-19 17:24:32 +02004860static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
Miao Xie897ca6e92010-10-26 20:57:29 -04004861{
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004862 int i;
4863 int num_pages;
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004864 int mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
Miao Xie897ca6e92010-10-26 20:57:29 -04004865
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004866 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e92010-10-26 20:57:29 -04004867
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004868 num_pages = num_extent_pages(eb);
4869 for (i = 0; i < num_pages; i++) {
4870 struct page *page = eb->pages[i];
Miao Xie897ca6e92010-10-26 20:57:29 -04004871
Forrest Liu5d2361d2015-02-09 17:31:45 +08004872 if (!page)
4873 continue;
4874 if (mapped)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004875 spin_lock(&page->mapping->private_lock);
Forrest Liu5d2361d2015-02-09 17:31:45 +08004876 /*
4877 * We do this since we'll remove the pages after we've
4878 * removed the eb from the radix tree, so we could race
4879 * and have this page now attached to the new eb. So
4880 * only clear page_private if it's still connected to
4881 * this eb.
4882 */
4883 if (PagePrivate(page) &&
4884 page->private == (unsigned long)eb) {
4885 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4886 BUG_ON(PageDirty(page));
4887 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004888 /*
Forrest Liu5d2361d2015-02-09 17:31:45 +08004889 * We need to make sure we haven't be attached
4890 * to a new eb.
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004891 */
Forrest Liu5d2361d2015-02-09 17:31:45 +08004892 ClearPagePrivate(page);
4893 set_page_private(page, 0);
4894 /* One for the page private */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004895 put_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004896 }
Forrest Liu5d2361d2015-02-09 17:31:45 +08004897
4898 if (mapped)
4899 spin_unlock(&page->mapping->private_lock);
4900
Nicholas D Steeves01327612016-05-19 21:18:45 -04004901 /* One for when we allocated the page */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004902 put_page(page);
Nikolay Borisovd64766f2018-06-27 16:38:22 +03004903 }
Miao Xie897ca6e92010-10-26 20:57:29 -04004904}
4905
4906/*
4907 * Helper for releasing the extent buffer.
4908 */
4909static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4910{
David Sterba55ac0132018-07-19 17:24:32 +02004911 btrfs_release_extent_buffer_pages(eb);
Miao Xie897ca6e92010-10-26 20:57:29 -04004912 __free_extent_buffer(eb);
4913}
4914
Josef Bacikf28491e2013-12-16 13:24:27 -05004915static struct extent_buffer *
4916__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
David Sterba23d79d82014-06-15 02:55:29 +02004917 unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004918{
4919 struct extent_buffer *eb = NULL;
4920
Michal Hockod1b5c562015-08-19 14:17:40 +02004921 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004922 eb->start = start;
4923 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004924 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004925 eb->bflags = 0;
4926 rwlock_init(&eb->lock);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004927 atomic_set(&eb->blocking_readers, 0);
David Sterba06297d82019-05-02 16:47:23 +02004928 eb->blocking_writers = 0;
David Sterbaed1b4ed2018-08-24 16:31:17 +02004929 eb->lock_nested = false;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004930 init_waitqueue_head(&eb->write_lock_wq);
4931 init_waitqueue_head(&eb->read_lock_wq);
4932
4933 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4934
4935 spin_lock_init(&eb->refs_lock);
4936 atomic_set(&eb->refs, 1);
4937 atomic_set(&eb->io_pages, 0);
4938
4939 /*
4940 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4941 */
4942 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4943 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4944 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4945
David Sterba843ccf92018-08-24 14:56:28 +02004946#ifdef CONFIG_BTRFS_DEBUG
David Sterbaf3dc24c2019-05-02 16:51:53 +02004947 eb->spinning_writers = 0;
David Sterbaafd495a2018-08-24 15:57:38 +02004948 atomic_set(&eb->spinning_readers, 0);
David Sterba5c9c7992018-08-24 16:15:51 +02004949 atomic_set(&eb->read_locks, 0);
David Sterba00801ae2019-05-02 16:53:47 +02004950 eb->write_locks = 0;
David Sterba843ccf92018-08-24 14:56:28 +02004951#endif
4952
Josef Bacikdb7f3432013-08-07 14:54:37 -04004953 return eb;
4954}
4955
4956struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4957{
David Sterbacc5e31a2018-03-01 18:20:27 +01004958 int i;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004959 struct page *p;
4960 struct extent_buffer *new;
David Sterbacc5e31a2018-03-01 18:20:27 +01004961 int num_pages = num_extent_pages(src);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004962
David Sterba3f556f72014-06-15 03:20:26 +02004963 new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004964 if (new == NULL)
4965 return NULL;
4966
4967 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004968 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004969 if (!p) {
4970 btrfs_release_extent_buffer(new);
4971 return NULL;
4972 }
4973 attach_extent_buffer_page(new, p);
4974 WARN_ON(PageDirty(p));
4975 SetPageUptodate(p);
4976 new->pages[i] = p;
David Sterbafba1acf2016-11-08 17:56:24 +01004977 copy_page(page_address(p), page_address(src->pages[i]));
Josef Bacikdb7f3432013-08-07 14:54:37 -04004978 }
4979
Josef Bacikdb7f3432013-08-07 14:54:37 -04004980 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
Nikolay Borisovb0132a32018-06-27 16:38:24 +03004981 set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004982
4983 return new;
4984}
4985
Omar Sandoval0f331222015-09-29 20:50:31 -07004986struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4987 u64 start, unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004988{
4989 struct extent_buffer *eb;
David Sterbacc5e31a2018-03-01 18:20:27 +01004990 int num_pages;
4991 int i;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004992
David Sterba3f556f72014-06-15 03:20:26 +02004993 eb = __alloc_extent_buffer(fs_info, start, len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004994 if (!eb)
4995 return NULL;
4996
David Sterba65ad0102018-06-29 10:56:49 +02004997 num_pages = num_extent_pages(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004998 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004999 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04005000 if (!eb->pages[i])
5001 goto err;
5002 }
5003 set_extent_buffer_uptodate(eb);
5004 btrfs_set_header_nritems(eb, 0);
Nikolay Borisovb0132a32018-06-27 16:38:24 +03005005 set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
Josef Bacikdb7f3432013-08-07 14:54:37 -04005006
5007 return eb;
5008err:
5009 for (; i > 0; i--)
5010 __free_page(eb->pages[i - 1]);
5011 __free_extent_buffer(eb);
5012 return NULL;
5013}
5014
Omar Sandoval0f331222015-09-29 20:50:31 -07005015struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
Jeff Mahoneyda170662016-06-15 09:22:56 -04005016 u64 start)
Omar Sandoval0f331222015-09-29 20:50:31 -07005017{
Jeff Mahoneyda170662016-06-15 09:22:56 -04005018 return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
Omar Sandoval0f331222015-09-29 20:50:31 -07005019}
5020
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005021static void check_buffer_tree_ref(struct extent_buffer *eb)
5022{
Chris Mason242e18c2013-01-29 17:49:37 -05005023 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005024 /* the ref bit is tricky. We have to make sure it is set
5025 * if we have the buffer dirty. Otherwise the
5026 * code to free a buffer can end up dropping a dirty
5027 * page
5028 *
5029 * Once the ref bit is set, it won't go away while the
5030 * buffer is dirty or in writeback, and it also won't
5031 * go away while we have the reference count on the
5032 * eb bumped.
5033 *
5034 * We can't just set the ref bit without bumping the
5035 * ref on the eb because free_extent_buffer might
5036 * see the ref bit and try to clear it. If this happens
5037 * free_extent_buffer might end up dropping our original
5038 * ref by mistake and freeing the page before we are able
5039 * to add one more ref.
5040 *
5041 * So bump the ref count first, then set the bit. If someone
5042 * beat us to it, drop the ref we added.
5043 */
Chris Mason242e18c2013-01-29 17:49:37 -05005044 refs = atomic_read(&eb->refs);
5045 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5046 return;
5047
Josef Bacik594831c2012-07-20 16:11:08 -04005048 spin_lock(&eb->refs_lock);
5049 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005050 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04005051 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005052}
5053
Mel Gorman2457aec2014-06-04 16:10:31 -07005054static void mark_extent_buffer_accessed(struct extent_buffer *eb,
5055 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04005056{
David Sterbacc5e31a2018-03-01 18:20:27 +01005057 int num_pages, i;
Josef Bacik5df42352012-03-15 18:24:42 -04005058
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005059 check_buffer_tree_ref(eb);
5060
David Sterba65ad0102018-06-29 10:56:49 +02005061 num_pages = num_extent_pages(eb);
Josef Bacik5df42352012-03-15 18:24:42 -04005062 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005063 struct page *p = eb->pages[i];
5064
Mel Gorman2457aec2014-06-04 16:10:31 -07005065 if (p != accessed)
5066 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04005067 }
5068}
5069
Josef Bacikf28491e2013-12-16 13:24:27 -05005070struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
5071 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005072{
5073 struct extent_buffer *eb;
5074
5075 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05005076 eb = radix_tree_lookup(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005077 start >> PAGE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005078 if (eb && atomic_inc_not_zero(&eb->refs)) {
5079 rcu_read_unlock();
Filipe Manana062c19e2015-04-23 11:28:48 +01005080 /*
5081 * Lock our eb's refs_lock to avoid races with
5082 * free_extent_buffer. When we get our eb it might be flagged
5083 * with EXTENT_BUFFER_STALE and another task running
5084 * free_extent_buffer might have seen that flag set,
5085 * eb->refs == 2, that the buffer isn't under IO (dirty and
5086 * writeback flags not set) and it's still in the tree (flag
5087 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
5088 * of decrementing the extent buffer's reference count twice.
5089 * So here we could race and increment the eb's reference count,
5090 * clear its stale flag, mark it as dirty and drop our reference
5091 * before the other task finishes executing free_extent_buffer,
5092 * which would later result in an attempt to free an extent
5093 * buffer that is dirty.
5094 */
5095 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
5096 spin_lock(&eb->refs_lock);
5097 spin_unlock(&eb->refs_lock);
5098 }
Mel Gorman2457aec2014-06-04 16:10:31 -07005099 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005100 return eb;
5101 }
5102 rcu_read_unlock();
5103
5104 return NULL;
5105}
5106
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005107#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5108struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
Jeff Mahoneyda170662016-06-15 09:22:56 -04005109 u64 start)
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005110{
5111 struct extent_buffer *eb, *exists = NULL;
5112 int ret;
5113
5114 eb = find_extent_buffer(fs_info, start);
5115 if (eb)
5116 return eb;
Jeff Mahoneyda170662016-06-15 09:22:56 -04005117 eb = alloc_dummy_extent_buffer(fs_info, start);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005118 if (!eb)
Dan Carpenterb6293c82019-12-03 14:24:58 +03005119 return ERR_PTR(-ENOMEM);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005120 eb->fs_info = fs_info;
5121again:
David Sterbae1860a72016-05-09 14:11:38 +02005122 ret = radix_tree_preload(GFP_NOFS);
Dan Carpenterb6293c82019-12-03 14:24:58 +03005123 if (ret) {
5124 exists = ERR_PTR(ret);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005125 goto free_eb;
Dan Carpenterb6293c82019-12-03 14:24:58 +03005126 }
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005127 spin_lock(&fs_info->buffer_lock);
5128 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005129 start >> PAGE_SHIFT, eb);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005130 spin_unlock(&fs_info->buffer_lock);
5131 radix_tree_preload_end();
5132 if (ret == -EEXIST) {
5133 exists = find_extent_buffer(fs_info, start);
5134 if (exists)
5135 goto free_eb;
5136 else
5137 goto again;
5138 }
5139 check_buffer_tree_ref(eb);
5140 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
5141
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005142 return eb;
5143free_eb:
5144 btrfs_release_extent_buffer(eb);
5145 return exists;
5146}
5147#endif
5148
Josef Bacikf28491e2013-12-16 13:24:27 -05005149struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02005150 u64 start)
Chris Masond1310b22008-01-24 16:13:08 -05005151{
Jeff Mahoneyda170662016-06-15 09:22:56 -04005152 unsigned long len = fs_info->nodesize;
David Sterbacc5e31a2018-03-01 18:20:27 +01005153 int num_pages;
5154 int i;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005155 unsigned long index = start >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005156 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04005157 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05005158 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05005159 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05005160 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005161 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05005162
Jeff Mahoneyda170662016-06-15 09:22:56 -04005163 if (!IS_ALIGNED(start, fs_info->sectorsize)) {
Liu Boc871b0f2016-06-06 12:01:23 -07005164 btrfs_err(fs_info, "bad tree block start %llu", start);
5165 return ERR_PTR(-EINVAL);
5166 }
5167
Josef Bacikf28491e2013-12-16 13:24:27 -05005168 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005169 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04005170 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04005171
David Sterba23d79d82014-06-15 02:55:29 +02005172 eb = __alloc_extent_buffer(fs_info, start, len);
Peter2b114d12008-04-01 11:21:40 -04005173 if (!eb)
Liu Boc871b0f2016-06-06 12:01:23 -07005174 return ERR_PTR(-ENOMEM);
Chris Masond1310b22008-01-24 16:13:08 -05005175
David Sterba65ad0102018-06-29 10:56:49 +02005176 num_pages = num_extent_pages(eb);
Chris Mason727011e2010-08-06 13:21:20 -04005177 for (i = 0; i < num_pages; i++, index++) {
Michal Hockod1b5c562015-08-19 14:17:40 +02005178 p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
Liu Boc871b0f2016-06-06 12:01:23 -07005179 if (!p) {
5180 exists = ERR_PTR(-ENOMEM);
Chris Mason6af118ce2008-07-22 11:18:07 -04005181 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07005182 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005183
5184 spin_lock(&mapping->private_lock);
5185 if (PagePrivate(p)) {
5186 /*
5187 * We could have already allocated an eb for this page
5188 * and attached one so lets see if we can get a ref on
5189 * the existing eb, and if we can we know it's good and
5190 * we can just return that one, else we know we can just
5191 * overwrite page->private.
5192 */
5193 exists = (struct extent_buffer *)p->private;
5194 if (atomic_inc_not_zero(&exists->refs)) {
5195 spin_unlock(&mapping->private_lock);
5196 unlock_page(p);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005197 put_page(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07005198 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005199 goto free_eb;
5200 }
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005201 exists = NULL;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005202
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005203 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005204 * Do this so attach doesn't complain and we need to
5205 * drop the ref the old guy had.
5206 */
5207 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005208 WARN_ON(PageDirty(p));
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005209 put_page(p);
Chris Masond1310b22008-01-24 16:13:08 -05005210 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005211 attach_extent_buffer_page(eb, p);
5212 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005213 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04005214 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05005215 if (!PageUptodate(p))
5216 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05005217
5218 /*
Nikolay Borisovb16d0112018-07-04 10:24:52 +03005219 * We can't unlock the pages just yet since the extent buffer
5220 * hasn't been properly inserted in the radix tree, this
5221 * opens a race with btree_releasepage which can free a page
5222 * while we are still filling in all pages for the buffer and
5223 * we could crash.
Chris Masoneb14ab82011-02-10 12:35:00 -05005224 */
Chris Masond1310b22008-01-24 16:13:08 -05005225 }
5226 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05005227 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05005228again:
David Sterbae1860a72016-05-09 14:11:38 +02005229 ret = radix_tree_preload(GFP_NOFS);
Liu Boc871b0f2016-06-06 12:01:23 -07005230 if (ret) {
5231 exists = ERR_PTR(ret);
Miao Xie19fe0a82010-10-26 20:57:29 -04005232 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07005233 }
Miao Xie19fe0a82010-10-26 20:57:29 -04005234
Josef Bacikf28491e2013-12-16 13:24:27 -05005235 spin_lock(&fs_info->buffer_lock);
5236 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005237 start >> PAGE_SHIFT, eb);
Josef Bacikf28491e2013-12-16 13:24:27 -05005238 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005239 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04005240 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005241 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005242 if (exists)
5243 goto free_eb;
5244 else
Josef Bacik115391d2012-03-09 09:51:43 -05005245 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04005246 }
Chris Mason6af118ce2008-07-22 11:18:07 -04005247 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005248 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005249 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05005250
5251 /*
Nikolay Borisovb16d0112018-07-04 10:24:52 +03005252 * Now it's safe to unlock the pages because any calls to
5253 * btree_releasepage will correctly detect that a page belongs to a
5254 * live buffer and won't free them prematurely.
Chris Masoneb14ab82011-02-10 12:35:00 -05005255 */
Nikolay Borisov28187ae2018-07-04 10:24:51 +03005256 for (i = 0; i < num_pages; i++)
5257 unlock_page(eb->pages[i]);
Chris Masond1310b22008-01-24 16:13:08 -05005258 return eb;
5259
Chris Mason6af118ce2008-07-22 11:18:07 -04005260free_eb:
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005261 WARN_ON(!atomic_dec_and_test(&eb->refs));
Chris Mason727011e2010-08-06 13:21:20 -04005262 for (i = 0; i < num_pages; i++) {
5263 if (eb->pages[i])
5264 unlock_page(eb->pages[i]);
5265 }
Chris Masoneb14ab82011-02-10 12:35:00 -05005266
Miao Xie897ca6e92010-10-26 20:57:29 -04005267 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005268 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05005269}
Chris Masond1310b22008-01-24 16:13:08 -05005270
Josef Bacik3083ee22012-03-09 16:01:49 -05005271static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
5272{
5273 struct extent_buffer *eb =
5274 container_of(head, struct extent_buffer, rcu_head);
5275
5276 __free_extent_buffer(eb);
5277}
5278
David Sterbaf7a52a42013-04-26 14:56:29 +00005279static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05005280{
Nikolay Borisov07e21c42018-06-27 16:38:23 +03005281 lockdep_assert_held(&eb->refs_lock);
5282
Josef Bacik3083ee22012-03-09 16:01:49 -05005283 WARN_ON(atomic_read(&eb->refs) == 0);
5284 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05005285 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005286 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05005287
Jan Schmidt815a51c2012-05-16 17:00:02 +02005288 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05005289
Josef Bacikf28491e2013-12-16 13:24:27 -05005290 spin_lock(&fs_info->buffer_lock);
5291 radix_tree_delete(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005292 eb->start >> PAGE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05005293 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005294 } else {
5295 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02005296 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005297
5298 /* Should be safe to release our pages at this point */
David Sterba55ac0132018-07-19 17:24:32 +02005299 btrfs_release_extent_buffer_pages(eb);
Josef Bacikbcb7e442015-03-16 17:38:02 -04005300#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
Nikolay Borisovb0132a32018-06-27 16:38:24 +03005301 if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
Josef Bacikbcb7e442015-03-16 17:38:02 -04005302 __free_extent_buffer(eb);
5303 return 1;
5304 }
5305#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05005306 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04005307 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05005308 }
5309 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04005310
5311 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05005312}
5313
Chris Masond1310b22008-01-24 16:13:08 -05005314void free_extent_buffer(struct extent_buffer *eb)
5315{
Chris Mason242e18c2013-01-29 17:49:37 -05005316 int refs;
5317 int old;
Chris Masond1310b22008-01-24 16:13:08 -05005318 if (!eb)
5319 return;
5320
Chris Mason242e18c2013-01-29 17:49:37 -05005321 while (1) {
5322 refs = atomic_read(&eb->refs);
Nikolay Borisov46cc7752018-10-15 17:04:01 +03005323 if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
5324 || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
5325 refs == 1))
Chris Mason242e18c2013-01-29 17:49:37 -05005326 break;
5327 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5328 if (old == refs)
5329 return;
5330 }
5331
Josef Bacik3083ee22012-03-09 16:01:49 -05005332 spin_lock(&eb->refs_lock);
5333 if (atomic_read(&eb->refs) == 2 &&
5334 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005335 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005336 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5337 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05005338
Josef Bacik3083ee22012-03-09 16:01:49 -05005339 /*
5340 * I know this is terrible, but it's temporary until we stop tracking
5341 * the uptodate bits and such for the extent buffers.
5342 */
David Sterbaf7a52a42013-04-26 14:56:29 +00005343 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005344}
Chris Masond1310b22008-01-24 16:13:08 -05005345
Josef Bacik3083ee22012-03-09 16:01:49 -05005346void free_extent_buffer_stale(struct extent_buffer *eb)
5347{
5348 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05005349 return;
5350
Josef Bacik3083ee22012-03-09 16:01:49 -05005351 spin_lock(&eb->refs_lock);
5352 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5353
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005354 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005355 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5356 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00005357 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005358}
5359
Chris Mason1d4284b2012-03-28 20:31:37 -04005360void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005361{
David Sterbacc5e31a2018-03-01 18:20:27 +01005362 int i;
5363 int num_pages;
Chris Masond1310b22008-01-24 16:13:08 -05005364 struct page *page;
5365
David Sterba65ad0102018-06-29 10:56:49 +02005366 num_pages = num_extent_pages(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005367
5368 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005369 page = eb->pages[i];
Chris Masonb9473432009-03-13 11:00:37 -04005370 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05005371 continue;
5372
Chris Masona61e6f22008-07-22 11:18:08 -04005373 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05005374 WARN_ON(!PagePrivate(page));
5375
Chris Masond1310b22008-01-24 16:13:08 -05005376 clear_page_dirty_for_io(page);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07005377 xa_lock_irq(&page->mapping->i_pages);
Matthew Wilcox0a943c62017-12-04 10:37:22 -05005378 if (!PageDirty(page))
5379 __xa_clear_mark(&page->mapping->i_pages,
5380 page_index(page), PAGECACHE_TAG_DIRTY);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07005381 xa_unlock_irq(&page->mapping->i_pages);
Chris Masonbf0da8c2011-11-04 12:29:37 -04005382 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04005383 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05005384 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005385 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05005386}
Chris Masond1310b22008-01-24 16:13:08 -05005387
Liu Boabb57ef2018-09-14 01:44:42 +08005388bool set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005389{
David Sterbacc5e31a2018-03-01 18:20:27 +01005390 int i;
5391 int num_pages;
Liu Boabb57ef2018-09-14 01:44:42 +08005392 bool was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005393
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005394 check_buffer_tree_ref(eb);
5395
Chris Masonb9473432009-03-13 11:00:37 -04005396 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005397
David Sterba65ad0102018-06-29 10:56:49 +02005398 num_pages = num_extent_pages(eb);
Josef Bacik3083ee22012-03-09 16:01:49 -05005399 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005400 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5401
Liu Boabb57ef2018-09-14 01:44:42 +08005402 if (!was_dirty)
5403 for (i = 0; i < num_pages; i++)
5404 set_page_dirty(eb->pages[i]);
Liu Bo51995c32018-09-14 01:46:08 +08005405
5406#ifdef CONFIG_BTRFS_DEBUG
5407 for (i = 0; i < num_pages; i++)
5408 ASSERT(PageDirty(eb->pages[i]));
5409#endif
5410
Chris Masonb9473432009-03-13 11:00:37 -04005411 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005412}
Chris Masond1310b22008-01-24 16:13:08 -05005413
David Sterba69ba3922015-12-03 13:08:59 +01005414void clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04005415{
David Sterbacc5e31a2018-03-01 18:20:27 +01005416 int i;
Chris Mason1259ab72008-05-12 13:39:03 -04005417 struct page *page;
David Sterbacc5e31a2018-03-01 18:20:27 +01005418 int num_pages;
Chris Mason1259ab72008-05-12 13:39:03 -04005419
Chris Masonb4ce94d2009-02-04 09:25:08 -05005420 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02005421 num_pages = num_extent_pages(eb);
Chris Mason1259ab72008-05-12 13:39:03 -04005422 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005423 page = eb->pages[i];
Chris Mason33958dc2008-07-30 10:29:12 -04005424 if (page)
5425 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04005426 }
Chris Mason1259ab72008-05-12 13:39:03 -04005427}
5428
David Sterba09c25a82015-12-03 13:08:59 +01005429void set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005430{
David Sterbacc5e31a2018-03-01 18:20:27 +01005431 int i;
Chris Masond1310b22008-01-24 16:13:08 -05005432 struct page *page;
David Sterbacc5e31a2018-03-01 18:20:27 +01005433 int num_pages;
Chris Masond1310b22008-01-24 16:13:08 -05005434
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005435 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
David Sterba65ad0102018-06-29 10:56:49 +02005436 num_pages = num_extent_pages(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005437 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005438 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005439 SetPageUptodate(page);
5440 }
Chris Masond1310b22008-01-24 16:13:08 -05005441}
Chris Masond1310b22008-01-24 16:13:08 -05005442
Nikolay Borisovc2ccfbc2019-04-10 17:24:40 +03005443int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05005444{
David Sterbacc5e31a2018-03-01 18:20:27 +01005445 int i;
Chris Masond1310b22008-01-24 16:13:08 -05005446 struct page *page;
5447 int err;
5448 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04005449 int locked_pages = 0;
5450 int all_uptodate = 1;
David Sterbacc5e31a2018-03-01 18:20:27 +01005451 int num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04005452 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005453 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04005454 unsigned long bio_flags = 0;
Nikolay Borisovc2ccfbc2019-04-10 17:24:40 +03005455 struct extent_io_tree *tree = &BTRFS_I(eb->fs_info->btree_inode)->io_tree;
Chris Masona86c12c2008-02-07 10:50:54 -05005456
Chris Masonb4ce94d2009-02-04 09:25:08 -05005457 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05005458 return 0;
5459
David Sterbaae6957e2020-02-05 19:09:30 +01005460 ASSERT(tree == &BTRFS_I(eb->pages[0]->mapping->host)->io_tree);
5461
David Sterba65ad0102018-06-29 10:56:49 +02005462 num_pages = num_extent_pages(eb);
Josef Bacik8436ea912016-09-02 15:40:03 -04005463 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005464 page = eb->pages[i];
Arne Jansenbb82ab82011-06-10 14:06:53 +02005465 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04005466 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04005467 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05005468 } else {
5469 lock_page(page);
5470 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005471 locked_pages++;
Liu Bo2571e732016-08-03 12:33:01 -07005472 }
5473 /*
5474 * We need to firstly lock all pages to make sure that
5475 * the uptodate bit of our pages won't be affected by
5476 * clear_extent_buffer_uptodate().
5477 */
Josef Bacik8436ea912016-09-02 15:40:03 -04005478 for (i = 0; i < num_pages; i++) {
Liu Bo2571e732016-08-03 12:33:01 -07005479 page = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04005480 if (!PageUptodate(page)) {
5481 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04005482 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04005483 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005484 }
Liu Bo2571e732016-08-03 12:33:01 -07005485
Chris Masonce9adaa2008-04-09 16:28:12 -04005486 if (all_uptodate) {
Josef Bacik8436ea912016-09-02 15:40:03 -04005487 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04005488 goto unlock_exit;
5489 }
5490
Filipe Manana656f30d2014-09-26 12:25:56 +01005491 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04005492 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005493 atomic_set(&eb->io_pages, num_reads);
Josef Bacik8436ea912016-09-02 15:40:03 -04005494 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005495 page = eb->pages[i];
Liu Bobaf863b2016-07-11 10:39:07 -07005496
Chris Masonce9adaa2008-04-09 16:28:12 -04005497 if (!PageUptodate(page)) {
Liu Bobaf863b2016-07-11 10:39:07 -07005498 if (ret) {
5499 atomic_dec(&eb->io_pages);
5500 unlock_page(page);
5501 continue;
5502 }
5503
Chris Masonf1885912008-04-09 16:28:12 -04005504 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05005505 err = __extent_read_full_page(tree, page,
David Sterba6af49db2017-06-23 04:09:57 +02005506 btree_get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005507 mirror_num, &bio_flags,
Mike Christie1f7ad752016-06-05 14:31:51 -05005508 REQ_META);
Liu Bobaf863b2016-07-11 10:39:07 -07005509 if (err) {
Chris Masond1310b22008-01-24 16:13:08 -05005510 ret = err;
Liu Bobaf863b2016-07-11 10:39:07 -07005511 /*
5512 * We use &bio in above __extent_read_full_page,
5513 * so we ensure that if it returns error, the
5514 * current page fails to add itself to bio and
5515 * it's been unlocked.
5516 *
5517 * We must dec io_pages by ourselves.
5518 */
5519 atomic_dec(&eb->io_pages);
5520 }
Chris Masond1310b22008-01-24 16:13:08 -05005521 } else {
5522 unlock_page(page);
5523 }
5524 }
5525
Jeff Mahoney355808c2011-10-03 23:23:14 -04005526 if (bio) {
Mike Christie1f7ad752016-06-05 14:31:51 -05005527 err = submit_one_bio(bio, mirror_num, bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005528 if (err)
5529 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005530 }
Chris Masona86c12c2008-02-07 10:50:54 -05005531
Arne Jansenbb82ab82011-06-10 14:06:53 +02005532 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005533 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005534
Josef Bacik8436ea912016-09-02 15:40:03 -04005535 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005536 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005537 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005538 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005539 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005540 }
Chris Masond3977122009-01-05 21:25:51 -05005541
Chris Masond1310b22008-01-24 16:13:08 -05005542 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005543
5544unlock_exit:
Chris Masond3977122009-01-05 21:25:51 -05005545 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04005546 locked_pages--;
Josef Bacik8436ea912016-09-02 15:40:03 -04005547 page = eb->pages[locked_pages];
5548 unlock_page(page);
Chris Masonce9adaa2008-04-09 16:28:12 -04005549 }
5550 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005551}
Chris Masond1310b22008-01-24 16:13:08 -05005552
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005553void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
5554 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005555{
5556 size_t cur;
5557 size_t offset;
5558 struct page *page;
5559 char *kaddr;
5560 char *dst = (char *)dstv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005561 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005562 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005563
Liu Bof716abd2017-08-09 11:10:16 -06005564 if (start + len > eb->len) {
5565 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5566 eb->start, eb->len, start, len);
5567 memset(dst, 0, len);
5568 return;
5569 }
Chris Masond1310b22008-01-24 16:13:08 -05005570
Johannes Thumshirn70730172018-12-05 15:23:03 +01005571 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005572
Chris Masond3977122009-01-05 21:25:51 -05005573 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005574 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005575
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005576 cur = min(len, (PAGE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005577 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005578 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005579
5580 dst += cur;
5581 len -= cur;
5582 offset = 0;
5583 i++;
5584 }
5585}
Chris Masond1310b22008-01-24 16:13:08 -05005586
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005587int read_extent_buffer_to_user(const struct extent_buffer *eb,
5588 void __user *dstv,
5589 unsigned long start, unsigned long len)
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005590{
5591 size_t cur;
5592 size_t offset;
5593 struct page *page;
5594 char *kaddr;
5595 char __user *dst = (char __user *)dstv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005596 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005597 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005598 int ret = 0;
5599
5600 WARN_ON(start > eb->len);
5601 WARN_ON(start + len > eb->start + eb->len);
5602
Johannes Thumshirn70730172018-12-05 15:23:03 +01005603 offset = offset_in_page(start_offset + start);
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005604
5605 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005606 page = eb->pages[i];
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005607
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005608 cur = min(len, (PAGE_SIZE - offset));
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005609 kaddr = page_address(page);
5610 if (copy_to_user(dst, kaddr + offset, cur)) {
5611 ret = -EFAULT;
5612 break;
5613 }
5614
5615 dst += cur;
5616 len -= cur;
5617 offset = 0;
5618 i++;
5619 }
5620
5621 return ret;
5622}
5623
Liu Bo415b35a2016-06-17 19:16:21 -07005624/*
5625 * return 0 if the item is found within a page.
5626 * return 1 if the item spans two pages.
5627 * return -EINVAL otherwise.
5628 */
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005629int map_private_extent_buffer(const struct extent_buffer *eb,
5630 unsigned long start, unsigned long min_len,
5631 char **map, unsigned long *map_start,
5632 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005633{
Johannes Thumshirncc2c39d2018-11-28 09:54:54 +01005634 size_t offset;
Chris Masond1310b22008-01-24 16:13:08 -05005635 char *kaddr;
5636 struct page *p;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005637 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005638 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005639 unsigned long end_i = (start_offset + start + min_len - 1) >>
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005640 PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005641
Liu Bof716abd2017-08-09 11:10:16 -06005642 if (start + min_len > eb->len) {
5643 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5644 eb->start, eb->len, start, min_len);
5645 return -EINVAL;
5646 }
5647
Chris Masond1310b22008-01-24 16:13:08 -05005648 if (i != end_i)
Liu Bo415b35a2016-06-17 19:16:21 -07005649 return 1;
Chris Masond1310b22008-01-24 16:13:08 -05005650
5651 if (i == 0) {
5652 offset = start_offset;
5653 *map_start = 0;
5654 } else {
5655 offset = 0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005656 *map_start = ((u64)i << PAGE_SHIFT) - start_offset;
Chris Masond1310b22008-01-24 16:13:08 -05005657 }
Chris Masond3977122009-01-05 21:25:51 -05005658
David Sterbafb85fc92014-07-31 01:03:53 +02005659 p = eb->pages[i];
Chris Masona6591712011-07-19 12:04:14 -04005660 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005661 *map = kaddr + offset;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005662 *map_len = PAGE_SIZE - offset;
Chris Masond1310b22008-01-24 16:13:08 -05005663 return 0;
5664}
Chris Masond1310b22008-01-24 16:13:08 -05005665
Jeff Mahoney1cbb1f42017-06-28 21:56:53 -06005666int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
5667 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005668{
5669 size_t cur;
5670 size_t offset;
5671 struct page *page;
5672 char *kaddr;
5673 char *ptr = (char *)ptrv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005674 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005675 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005676 int ret = 0;
5677
5678 WARN_ON(start > eb->len);
5679 WARN_ON(start + len > eb->start + eb->len);
5680
Johannes Thumshirn70730172018-12-05 15:23:03 +01005681 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005682
Chris Masond3977122009-01-05 21:25:51 -05005683 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005684 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005685
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005686 cur = min(len, (PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005687
Chris Masona6591712011-07-19 12:04:14 -04005688 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005689 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005690 if (ret)
5691 break;
5692
5693 ptr += cur;
5694 len -= cur;
5695 offset = 0;
5696 i++;
5697 }
5698 return ret;
5699}
Chris Masond1310b22008-01-24 16:13:08 -05005700
David Sterbaf157bf72016-11-09 17:43:38 +01005701void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb,
5702 const void *srcv)
5703{
5704 char *kaddr;
5705
5706 WARN_ON(!PageUptodate(eb->pages[0]));
5707 kaddr = page_address(eb->pages[0]);
5708 memcpy(kaddr + offsetof(struct btrfs_header, chunk_tree_uuid), srcv,
5709 BTRFS_FSID_SIZE);
5710}
5711
5712void write_extent_buffer_fsid(struct extent_buffer *eb, const void *srcv)
5713{
5714 char *kaddr;
5715
5716 WARN_ON(!PageUptodate(eb->pages[0]));
5717 kaddr = page_address(eb->pages[0]);
5718 memcpy(kaddr + offsetof(struct btrfs_header, fsid), srcv,
5719 BTRFS_FSID_SIZE);
5720}
5721
Chris Masond1310b22008-01-24 16:13:08 -05005722void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5723 unsigned long start, unsigned long len)
5724{
5725 size_t cur;
5726 size_t offset;
5727 struct page *page;
5728 char *kaddr;
5729 char *src = (char *)srcv;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005730 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005731 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005732
5733 WARN_ON(start > eb->len);
5734 WARN_ON(start + len > eb->start + eb->len);
5735
Johannes Thumshirn70730172018-12-05 15:23:03 +01005736 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005737
Chris Masond3977122009-01-05 21:25:51 -05005738 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005739 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005740 WARN_ON(!PageUptodate(page));
5741
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005742 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005743 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005744 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005745
5746 src += cur;
5747 len -= cur;
5748 offset = 0;
5749 i++;
5750 }
5751}
Chris Masond1310b22008-01-24 16:13:08 -05005752
David Sterbab159fa22016-11-08 18:09:03 +01005753void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start,
5754 unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005755{
5756 size_t cur;
5757 size_t offset;
5758 struct page *page;
5759 char *kaddr;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005760 size_t start_offset = offset_in_page(eb->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005761 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005762
5763 WARN_ON(start > eb->len);
5764 WARN_ON(start + len > eb->start + eb->len);
5765
Johannes Thumshirn70730172018-12-05 15:23:03 +01005766 offset = offset_in_page(start_offset + start);
Chris Masond1310b22008-01-24 16:13:08 -05005767
Chris Masond3977122009-01-05 21:25:51 -05005768 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005769 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005770 WARN_ON(!PageUptodate(page));
5771
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005772 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005773 kaddr = page_address(page);
David Sterbab159fa22016-11-08 18:09:03 +01005774 memset(kaddr + offset, 0, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005775
5776 len -= cur;
5777 offset = 0;
5778 i++;
5779 }
5780}
Chris Masond1310b22008-01-24 16:13:08 -05005781
David Sterba58e80122016-11-08 18:30:31 +01005782void copy_extent_buffer_full(struct extent_buffer *dst,
5783 struct extent_buffer *src)
5784{
5785 int i;
David Sterbacc5e31a2018-03-01 18:20:27 +01005786 int num_pages;
David Sterba58e80122016-11-08 18:30:31 +01005787
5788 ASSERT(dst->len == src->len);
5789
David Sterba65ad0102018-06-29 10:56:49 +02005790 num_pages = num_extent_pages(dst);
David Sterba58e80122016-11-08 18:30:31 +01005791 for (i = 0; i < num_pages; i++)
5792 copy_page(page_address(dst->pages[i]),
5793 page_address(src->pages[i]));
5794}
5795
Chris Masond1310b22008-01-24 16:13:08 -05005796void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5797 unsigned long dst_offset, unsigned long src_offset,
5798 unsigned long len)
5799{
5800 u64 dst_len = dst->len;
5801 size_t cur;
5802 size_t offset;
5803 struct page *page;
5804 char *kaddr;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005805 size_t start_offset = offset_in_page(dst->start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005806 unsigned long i = (start_offset + dst_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005807
5808 WARN_ON(src->len != dst_len);
5809
Johannes Thumshirn70730172018-12-05 15:23:03 +01005810 offset = offset_in_page(start_offset + dst_offset);
Chris Masond1310b22008-01-24 16:13:08 -05005811
Chris Masond3977122009-01-05 21:25:51 -05005812 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005813 page = dst->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005814 WARN_ON(!PageUptodate(page));
5815
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005816 cur = min(len, (unsigned long)(PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005817
Chris Masona6591712011-07-19 12:04:14 -04005818 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005819 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005820
5821 src_offset += cur;
5822 len -= cur;
5823 offset = 0;
5824 i++;
5825 }
5826}
Chris Masond1310b22008-01-24 16:13:08 -05005827
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005828/*
5829 * eb_bitmap_offset() - calculate the page and offset of the byte containing the
5830 * given bit number
5831 * @eb: the extent buffer
5832 * @start: offset of the bitmap item in the extent buffer
5833 * @nr: bit number
5834 * @page_index: return index of the page in the extent buffer that contains the
5835 * given bit number
5836 * @page_offset: return offset into the page given by page_index
5837 *
5838 * This helper hides the ugliness of finding the byte in an extent buffer which
5839 * contains a given bit.
5840 */
5841static inline void eb_bitmap_offset(struct extent_buffer *eb,
5842 unsigned long start, unsigned long nr,
5843 unsigned long *page_index,
5844 size_t *page_offset)
5845{
Johannes Thumshirn70730172018-12-05 15:23:03 +01005846 size_t start_offset = offset_in_page(eb->start);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005847 size_t byte_offset = BIT_BYTE(nr);
5848 size_t offset;
5849
5850 /*
5851 * The byte we want is the offset of the extent buffer + the offset of
5852 * the bitmap item in the extent buffer + the offset of the byte in the
5853 * bitmap item.
5854 */
5855 offset = start_offset + start + byte_offset;
5856
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005857 *page_index = offset >> PAGE_SHIFT;
Johannes Thumshirn70730172018-12-05 15:23:03 +01005858 *page_offset = offset_in_page(offset);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005859}
5860
5861/**
5862 * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
5863 * @eb: the extent buffer
5864 * @start: offset of the bitmap item in the extent buffer
5865 * @nr: bit number to test
5866 */
5867int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start,
5868 unsigned long nr)
5869{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005870 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005871 struct page *page;
5872 unsigned long i;
5873 size_t offset;
5874
5875 eb_bitmap_offset(eb, start, nr, &i, &offset);
5876 page = eb->pages[i];
5877 WARN_ON(!PageUptodate(page));
5878 kaddr = page_address(page);
5879 return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
5880}
5881
5882/**
5883 * extent_buffer_bitmap_set - set an area of a bitmap
5884 * @eb: the extent buffer
5885 * @start: offset of the bitmap item in the extent buffer
5886 * @pos: bit number of the first bit
5887 * @len: number of bits to set
5888 */
5889void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start,
5890 unsigned long pos, unsigned long len)
5891{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005892 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005893 struct page *page;
5894 unsigned long i;
5895 size_t offset;
5896 const unsigned int size = pos + len;
5897 int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005898 u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005899
5900 eb_bitmap_offset(eb, start, pos, &i, &offset);
5901 page = eb->pages[i];
5902 WARN_ON(!PageUptodate(page));
5903 kaddr = page_address(page);
5904
5905 while (len >= bits_to_set) {
5906 kaddr[offset] |= mask_to_set;
5907 len -= bits_to_set;
5908 bits_to_set = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005909 mask_to_set = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005910 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005911 offset = 0;
5912 page = eb->pages[++i];
5913 WARN_ON(!PageUptodate(page));
5914 kaddr = page_address(page);
5915 }
5916 }
5917 if (len) {
5918 mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
5919 kaddr[offset] |= mask_to_set;
5920 }
5921}
5922
5923
5924/**
5925 * extent_buffer_bitmap_clear - clear an area of a bitmap
5926 * @eb: the extent buffer
5927 * @start: offset of the bitmap item in the extent buffer
5928 * @pos: bit number of the first bit
5929 * @len: number of bits to clear
5930 */
5931void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start,
5932 unsigned long pos, unsigned long len)
5933{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005934 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005935 struct page *page;
5936 unsigned long i;
5937 size_t offset;
5938 const unsigned int size = pos + len;
5939 int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005940 u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005941
5942 eb_bitmap_offset(eb, start, pos, &i, &offset);
5943 page = eb->pages[i];
5944 WARN_ON(!PageUptodate(page));
5945 kaddr = page_address(page);
5946
5947 while (len >= bits_to_clear) {
5948 kaddr[offset] &= ~mask_to_clear;
5949 len -= bits_to_clear;
5950 bits_to_clear = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005951 mask_to_clear = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005952 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005953 offset = 0;
5954 page = eb->pages[++i];
5955 WARN_ON(!PageUptodate(page));
5956 kaddr = page_address(page);
5957 }
5958 }
5959 if (len) {
5960 mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
5961 kaddr[offset] &= ~mask_to_clear;
5962 }
5963}
5964
Sergei Trofimovich33872062011-04-11 21:52:52 +00005965static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5966{
5967 unsigned long distance = (src > dst) ? src - dst : dst - src;
5968 return distance < len;
5969}
5970
Chris Masond1310b22008-01-24 16:13:08 -05005971static void copy_pages(struct page *dst_page, struct page *src_page,
5972 unsigned long dst_off, unsigned long src_off,
5973 unsigned long len)
5974{
Chris Masona6591712011-07-19 12:04:14 -04005975 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005976 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005977 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005978
Sergei Trofimovich33872062011-04-11 21:52:52 +00005979 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005980 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005981 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005982 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005983 if (areas_overlap(src_off, dst_off, len))
5984 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005985 }
Chris Masond1310b22008-01-24 16:13:08 -05005986
Chris Mason727011e2010-08-06 13:21:20 -04005987 if (must_memmove)
5988 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5989 else
5990 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005991}
5992
5993void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5994 unsigned long src_offset, unsigned long len)
5995{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005996 struct btrfs_fs_info *fs_info = dst->fs_info;
Chris Masond1310b22008-01-24 16:13:08 -05005997 size_t cur;
5998 size_t dst_off_in_page;
5999 size_t src_off_in_page;
Johannes Thumshirn70730172018-12-05 15:23:03 +01006000 size_t start_offset = offset_in_page(dst->start);
Chris Masond1310b22008-01-24 16:13:08 -05006001 unsigned long dst_i;
6002 unsigned long src_i;
6003
6004 if (src_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006005 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04006006 "memmove bogus src_offset %lu move len %lu dst len %lu",
6007 src_offset, len, dst->len);
Arnd Bergmann290342f2019-03-25 14:02:25 +01006008 BUG();
Chris Masond1310b22008-01-24 16:13:08 -05006009 }
6010 if (dst_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006011 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04006012 "memmove bogus dst_offset %lu move len %lu dst len %lu",
6013 dst_offset, len, dst->len);
Arnd Bergmann290342f2019-03-25 14:02:25 +01006014 BUG();
Chris Masond1310b22008-01-24 16:13:08 -05006015 }
6016
Chris Masond3977122009-01-05 21:25:51 -05006017 while (len > 0) {
Johannes Thumshirn70730172018-12-05 15:23:03 +01006018 dst_off_in_page = offset_in_page(start_offset + dst_offset);
6019 src_off_in_page = offset_in_page(start_offset + src_offset);
Chris Masond1310b22008-01-24 16:13:08 -05006020
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03006021 dst_i = (start_offset + dst_offset) >> PAGE_SHIFT;
6022 src_i = (start_offset + src_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05006023
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03006024 cur = min(len, (unsigned long)(PAGE_SIZE -
Chris Masond1310b22008-01-24 16:13:08 -05006025 src_off_in_page));
6026 cur = min_t(unsigned long, cur,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03006027 (unsigned long)(PAGE_SIZE - dst_off_in_page));
Chris Masond1310b22008-01-24 16:13:08 -05006028
David Sterbafb85fc92014-07-31 01:03:53 +02006029 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05006030 dst_off_in_page, src_off_in_page, cur);
6031
6032 src_offset += cur;
6033 dst_offset += cur;
6034 len -= cur;
6035 }
6036}
Chris Masond1310b22008-01-24 16:13:08 -05006037
6038void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
6039 unsigned long src_offset, unsigned long len)
6040{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006041 struct btrfs_fs_info *fs_info = dst->fs_info;
Chris Masond1310b22008-01-24 16:13:08 -05006042 size_t cur;
6043 size_t dst_off_in_page;
6044 size_t src_off_in_page;
6045 unsigned long dst_end = dst_offset + len - 1;
6046 unsigned long src_end = src_offset + len - 1;
Johannes Thumshirn70730172018-12-05 15:23:03 +01006047 size_t start_offset = offset_in_page(dst->start);
Chris Masond1310b22008-01-24 16:13:08 -05006048 unsigned long dst_i;
6049 unsigned long src_i;
6050
6051 if (src_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006052 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04006053 "memmove bogus src_offset %lu move len %lu len %lu",
6054 src_offset, len, dst->len);
Arnd Bergmann290342f2019-03-25 14:02:25 +01006055 BUG();
Chris Masond1310b22008-01-24 16:13:08 -05006056 }
6057 if (dst_offset + len > dst->len) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006058 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04006059 "memmove bogus dst_offset %lu move len %lu len %lu",
6060 dst_offset, len, dst->len);
Arnd Bergmann290342f2019-03-25 14:02:25 +01006061 BUG();
Chris Masond1310b22008-01-24 16:13:08 -05006062 }
Chris Mason727011e2010-08-06 13:21:20 -04006063 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05006064 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
6065 return;
6066 }
Chris Masond3977122009-01-05 21:25:51 -05006067 while (len > 0) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03006068 dst_i = (start_offset + dst_end) >> PAGE_SHIFT;
6069 src_i = (start_offset + src_end) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05006070
Johannes Thumshirn70730172018-12-05 15:23:03 +01006071 dst_off_in_page = offset_in_page(start_offset + dst_end);
6072 src_off_in_page = offset_in_page(start_offset + src_end);
Chris Masond1310b22008-01-24 16:13:08 -05006073
6074 cur = min_t(unsigned long, len, src_off_in_page + 1);
6075 cur = min(cur, dst_off_in_page + 1);
David Sterbafb85fc92014-07-31 01:03:53 +02006076 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05006077 dst_off_in_page - cur + 1,
6078 src_off_in_page - cur + 1, cur);
6079
6080 dst_end -= cur;
6081 src_end -= cur;
6082 len -= cur;
6083 }
6084}
Chris Mason6af118ce2008-07-22 11:18:07 -04006085
David Sterbaf7a52a42013-04-26 14:56:29 +00006086int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04006087{
Chris Mason6af118ce2008-07-22 11:18:07 -04006088 struct extent_buffer *eb;
Miao Xie897ca6e92010-10-26 20:57:29 -04006089
Miao Xie19fe0a82010-10-26 20:57:29 -04006090 /*
Nicholas D Steeves01327612016-05-19 21:18:45 -04006091 * We need to make sure nobody is attaching this page to an eb right
Josef Bacik3083ee22012-03-09 16:01:49 -05006092 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04006093 */
Josef Bacik3083ee22012-03-09 16:01:49 -05006094 spin_lock(&page->mapping->private_lock);
6095 if (!PagePrivate(page)) {
6096 spin_unlock(&page->mapping->private_lock);
6097 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04006098 }
6099
Josef Bacik3083ee22012-03-09 16:01:49 -05006100 eb = (struct extent_buffer *)page->private;
6101 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04006102
Josef Bacik0b32f4b2012-03-13 09:38:00 -04006103 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05006104 * This is a little awful but should be ok, we need to make sure that
6105 * the eb doesn't disappear out from under us while we're looking at
6106 * this page.
6107 */
6108 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04006109 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05006110 spin_unlock(&eb->refs_lock);
6111 spin_unlock(&page->mapping->private_lock);
6112 return 0;
6113 }
6114 spin_unlock(&page->mapping->private_lock);
6115
Josef Bacik3083ee22012-03-09 16:01:49 -05006116 /*
6117 * If tree ref isn't set then we know the ref on this eb is a real ref,
6118 * so just return, this page will likely be freed soon anyway.
6119 */
6120 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
6121 spin_unlock(&eb->refs_lock);
6122 return 0;
6123 }
Josef Bacik3083ee22012-03-09 16:01:49 -05006124
David Sterbaf7a52a42013-04-26 14:56:29 +00006125 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04006126}