blob: 5116c2a1f0f9e96af06ac1a749df51091afad661 [file] [log] [blame]
David Sterbac1d7c512018-04-03 19:23:33 +02001// SPDX-License-Identifier: GPL-2.0
Chris Mason6cbd5572007-06-12 09:07:21 -04002/*
Chris Masond352ac62008-09-29 15:18:18 -04003 * Copyright (C) 2007,2008 Oracle. All rights reserved.
Chris Mason6cbd5572007-06-12 09:07:21 -04004 */
5
Chris Masona6b6e752007-10-15 16:22:39 -04006#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09007#include <linux/slab.h>
Jan Schmidtbd989ba2012-05-16 17:18:50 +02008#include <linux/rbtree.h>
David Sterbaadf02122017-05-31 19:44:31 +02009#include <linux/mm.h>
Chris Masoneb60cea2007-02-02 09:18:22 -050010#include "ctree.h"
11#include "disk-io.h"
Chris Mason7f5c1512007-03-23 15:56:19 -040012#include "transaction.h"
Chris Mason5f39d392007-10-15 16:14:19 -040013#include "print-tree.h"
Chris Mason925baed2008-06-25 16:01:30 -040014#include "locking.h"
Nikolay Borisovde37aa52018-10-30 16:43:24 +020015#include "volumes.h"
Qu Wenruof616f5c2019-01-23 15:15:17 +080016#include "qgroup.h"
Chris Mason9a8dd152007-02-23 08:38:36 -050017
Chris Masone089f052007-03-16 16:20:31 -040018static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root
19 *root, struct btrfs_path *path, int level);
Omar Sandoval310712b2017-01-17 23:24:37 -080020static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root,
21 const struct btrfs_key *ins_key, struct btrfs_path *path,
22 int data_size, int extend);
Chris Mason5f39d392007-10-15 16:14:19 -040023static int push_node_left(struct btrfs_trans_handle *trans,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040024 struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -040025 struct extent_buffer *src, int empty);
Chris Mason5f39d392007-10-15 16:14:19 -040026static int balance_node_right(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -040027 struct extent_buffer *dst_buf,
28 struct extent_buffer *src_buf);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +000029static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
30 int level, int slot);
Chris Masond97e63b2007-02-20 16:40:44 -050031
Chris Mason2c90e5d2007-04-02 10:50:19 -040032struct btrfs_path *btrfs_alloc_path(void)
33{
Masahiro Yamadae2c89902016-09-13 04:35:52 +090034 return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS);
Chris Mason2c90e5d2007-04-02 10:50:19 -040035}
36
Chris Masonb4ce94d2009-02-04 09:25:08 -050037/*
38 * set all locked nodes in the path to blocking locks. This should
39 * be done before scheduling
40 */
41noinline void btrfs_set_path_blocking(struct btrfs_path *p)
42{
43 int i;
44 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbd681512011-07-16 15:23:14 -040045 if (!p->nodes[i] || !p->locks[i])
46 continue;
David Sterba766ece52019-01-23 18:07:14 +010047 /*
48 * If we currently have a spinning reader or writer lock this
49 * will bump the count of blocking holders and drop the
50 * spinlock.
51 */
52 if (p->locks[i] == BTRFS_READ_LOCK) {
53 btrfs_set_lock_blocking_read(p->nodes[i]);
Chris Masonbd681512011-07-16 15:23:14 -040054 p->locks[i] = BTRFS_READ_LOCK_BLOCKING;
David Sterba766ece52019-01-23 18:07:14 +010055 } else if (p->locks[i] == BTRFS_WRITE_LOCK) {
56 btrfs_set_lock_blocking_write(p->nodes[i]);
Chris Masonbd681512011-07-16 15:23:14 -040057 p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING;
David Sterba766ece52019-01-23 18:07:14 +010058 }
Chris Masonb4ce94d2009-02-04 09:25:08 -050059 }
60}
61
Chris Masond352ac62008-09-29 15:18:18 -040062/* this also releases the path */
Chris Mason2c90e5d2007-04-02 10:50:19 -040063void btrfs_free_path(struct btrfs_path *p)
64{
Jesper Juhlff175d52010-12-25 21:22:30 +000065 if (!p)
66 return;
David Sterbab3b4aa72011-04-21 01:20:15 +020067 btrfs_release_path(p);
Chris Mason2c90e5d2007-04-02 10:50:19 -040068 kmem_cache_free(btrfs_path_cachep, p);
69}
70
Chris Masond352ac62008-09-29 15:18:18 -040071/*
72 * path release drops references on the extent buffers in the path
73 * and it drops any locks held by this path
74 *
75 * It is safe to call this on paths that no locks or extent buffers held.
76 */
David Sterbab3b4aa72011-04-21 01:20:15 +020077noinline void btrfs_release_path(struct btrfs_path *p)
Chris Masoneb60cea2007-02-02 09:18:22 -050078{
79 int i;
Chris Masona2135012008-06-25 16:01:30 -040080
Chris Mason234b63a2007-03-13 10:46:10 -040081 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Mason3f157a22008-06-25 16:01:31 -040082 p->slots[i] = 0;
Chris Masoneb60cea2007-02-02 09:18:22 -050083 if (!p->nodes[i])
Chris Mason925baed2008-06-25 16:01:30 -040084 continue;
85 if (p->locks[i]) {
Chris Masonbd681512011-07-16 15:23:14 -040086 btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -040087 p->locks[i] = 0;
88 }
Chris Mason5f39d392007-10-15 16:14:19 -040089 free_extent_buffer(p->nodes[i]);
Chris Mason3f157a22008-06-25 16:01:31 -040090 p->nodes[i] = NULL;
Chris Masoneb60cea2007-02-02 09:18:22 -050091 }
92}
93
Chris Masond352ac62008-09-29 15:18:18 -040094/*
95 * safely gets a reference on the root node of a tree. A lock
96 * is not taken, so a concurrent writer may put a different node
97 * at the root of the tree. See btrfs_lock_root_node for the
98 * looping required.
99 *
100 * The extent buffer returned by this has a reference taken, so
101 * it won't disappear. It may stop being the root of the tree
102 * at any time because there are no locks held.
103 */
Chris Mason925baed2008-06-25 16:01:30 -0400104struct extent_buffer *btrfs_root_node(struct btrfs_root *root)
105{
106 struct extent_buffer *eb;
Chris Mason240f62c2011-03-23 14:54:42 -0400107
Josef Bacik3083ee22012-03-09 16:01:49 -0500108 while (1) {
109 rcu_read_lock();
110 eb = rcu_dereference(root->node);
111
112 /*
113 * RCU really hurts here, we could free up the root node because
Nicholas D Steeves01327612016-05-19 21:18:45 -0400114 * it was COWed but we may not get the new root node yet so do
Josef Bacik3083ee22012-03-09 16:01:49 -0500115 * the inc_not_zero dance and if it doesn't work then
116 * synchronize_rcu and try again.
117 */
118 if (atomic_inc_not_zero(&eb->refs)) {
119 rcu_read_unlock();
120 break;
121 }
122 rcu_read_unlock();
123 synchronize_rcu();
124 }
Chris Mason925baed2008-06-25 16:01:30 -0400125 return eb;
126}
127
Chris Masond352ac62008-09-29 15:18:18 -0400128/* loop around taking references on and locking the root node of the
129 * tree until you end up with a lock on the root. A locked buffer
130 * is returned, with a reference held.
131 */
Chris Mason925baed2008-06-25 16:01:30 -0400132struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root)
133{
134 struct extent_buffer *eb;
135
Chris Masond3977122009-01-05 21:25:51 -0500136 while (1) {
Chris Mason925baed2008-06-25 16:01:30 -0400137 eb = btrfs_root_node(root);
138 btrfs_tree_lock(eb);
Chris Mason240f62c2011-03-23 14:54:42 -0400139 if (eb == root->node)
Chris Mason925baed2008-06-25 16:01:30 -0400140 break;
Chris Mason925baed2008-06-25 16:01:30 -0400141 btrfs_tree_unlock(eb);
142 free_extent_buffer(eb);
143 }
144 return eb;
145}
146
Chris Masonbd681512011-07-16 15:23:14 -0400147/* loop around taking references on and locking the root node of the
148 * tree until you end up with a lock on the root. A locked buffer
149 * is returned, with a reference held.
150 */
Josef Bacik84f7d8e2017-09-29 15:43:49 -0400151struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root)
Chris Masonbd681512011-07-16 15:23:14 -0400152{
153 struct extent_buffer *eb;
154
155 while (1) {
156 eb = btrfs_root_node(root);
157 btrfs_tree_read_lock(eb);
158 if (eb == root->node)
159 break;
160 btrfs_tree_read_unlock(eb);
161 free_extent_buffer(eb);
162 }
163 return eb;
164}
165
Chris Masond352ac62008-09-29 15:18:18 -0400166/* cowonly root (everything not a reference counted cow subvolume), just get
167 * put onto a simple dirty list. transaction.c walks this to make sure they
168 * get properly updated on disk.
169 */
Chris Mason0b86a832008-03-24 15:01:56 -0400170static void add_root_to_dirty_list(struct btrfs_root *root)
171{
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400172 struct btrfs_fs_info *fs_info = root->fs_info;
173
Josef Bacike7070be2014-12-16 08:54:43 -0800174 if (test_bit(BTRFS_ROOT_DIRTY, &root->state) ||
175 !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state))
176 return;
177
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400178 spin_lock(&fs_info->trans_lock);
Josef Bacike7070be2014-12-16 08:54:43 -0800179 if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) {
180 /* Want the extent tree to be the last on the list */
Misono Tomohiro4fd786e2018-08-06 14:25:24 +0900181 if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID)
Josef Bacike7070be2014-12-16 08:54:43 -0800182 list_move_tail(&root->dirty_list,
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400183 &fs_info->dirty_cowonly_roots);
Josef Bacike7070be2014-12-16 08:54:43 -0800184 else
185 list_move(&root->dirty_list,
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400186 &fs_info->dirty_cowonly_roots);
Chris Mason0b86a832008-03-24 15:01:56 -0400187 }
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400188 spin_unlock(&fs_info->trans_lock);
Chris Mason0b86a832008-03-24 15:01:56 -0400189}
190
Chris Masond352ac62008-09-29 15:18:18 -0400191/*
192 * used by snapshot creation to make a copy of a root for a tree with
193 * a given objectid. The buffer with the new root node is returned in
194 * cow_ret, and this func returns zero on success or a negative error code.
195 */
Chris Masonbe20aa92007-12-17 20:14:01 -0500196int btrfs_copy_root(struct btrfs_trans_handle *trans,
197 struct btrfs_root *root,
198 struct extent_buffer *buf,
199 struct extent_buffer **cow_ret, u64 new_root_objectid)
200{
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400201 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Masonbe20aa92007-12-17 20:14:01 -0500202 struct extent_buffer *cow;
Chris Masonbe20aa92007-12-17 20:14:01 -0500203 int ret = 0;
204 int level;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400205 struct btrfs_disk_key disk_key;
Chris Masonbe20aa92007-12-17 20:14:01 -0500206
Miao Xie27cdeb72014-04-02 19:51:05 +0800207 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400208 trans->transid != fs_info->running_transaction->transid);
Miao Xie27cdeb72014-04-02 19:51:05 +0800209 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
210 trans->transid != root->last_trans);
Chris Masonbe20aa92007-12-17 20:14:01 -0500211
212 level = btrfs_header_level(buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400213 if (level == 0)
214 btrfs_item_key(buf, &disk_key, 0);
215 else
216 btrfs_node_key(buf, &disk_key, 0);
Zheng Yan31840ae2008-09-23 13:14:14 -0400217
David Sterba4d75f8a2014-06-15 01:54:12 +0200218 cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid,
219 &disk_key, level, buf->start, 0);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400220 if (IS_ERR(cow))
Chris Masonbe20aa92007-12-17 20:14:01 -0500221 return PTR_ERR(cow);
222
David Sterba58e80122016-11-08 18:30:31 +0100223 copy_extent_buffer_full(cow, buf);
Chris Masonbe20aa92007-12-17 20:14:01 -0500224 btrfs_set_header_bytenr(cow, cow->start);
225 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400226 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
227 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
228 BTRFS_HEADER_FLAG_RELOC);
229 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
230 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
231 else
232 btrfs_set_header_owner(cow, new_root_objectid);
Chris Masonbe20aa92007-12-17 20:14:01 -0500233
Nikolay Borisovde37aa52018-10-30 16:43:24 +0200234 write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid);
Yan Zheng2b820322008-11-17 21:11:30 -0500235
Chris Masonbe20aa92007-12-17 20:14:01 -0500236 WARN_ON(btrfs_header_generation(buf) > trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400237 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -0700238 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400239 else
Josef Bacike339a6b2014-07-02 10:54:25 -0700240 ret = btrfs_inc_ref(trans, root, cow, 0);
Chris Mason4aec2b52007-12-18 16:25:45 -0500241
Chris Masonbe20aa92007-12-17 20:14:01 -0500242 if (ret)
243 return ret;
244
245 btrfs_mark_buffer_dirty(cow);
246 *cow_ret = cow;
247 return 0;
248}
249
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200250enum mod_log_op {
251 MOD_LOG_KEY_REPLACE,
252 MOD_LOG_KEY_ADD,
253 MOD_LOG_KEY_REMOVE,
254 MOD_LOG_KEY_REMOVE_WHILE_FREEING,
255 MOD_LOG_KEY_REMOVE_WHILE_MOVING,
256 MOD_LOG_MOVE_KEYS,
257 MOD_LOG_ROOT_REPLACE,
258};
259
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200260struct tree_mod_root {
261 u64 logical;
262 u8 level;
263};
264
265struct tree_mod_elem {
266 struct rb_node node;
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530267 u64 logical;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200268 u64 seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200269 enum mod_log_op op;
270
271 /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */
272 int slot;
273
274 /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */
275 u64 generation;
276
277 /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */
278 struct btrfs_disk_key key;
279 u64 blockptr;
280
281 /* this is used for op == MOD_LOG_MOVE_KEYS */
David Sterbab6dfa352018-03-05 15:31:18 +0100282 struct {
283 int dst_slot;
284 int nr_items;
285 } move;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200286
287 /* this is used for op == MOD_LOG_ROOT_REPLACE */
288 struct tree_mod_root old_root;
289};
290
Jan Schmidt097b8a72012-06-21 11:08:04 +0200291/*
Josef Bacikfcebe452014-05-13 17:30:47 -0700292 * Pull a new tree mod seq number for our operation.
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000293 */
Josef Bacikfcebe452014-05-13 17:30:47 -0700294static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info)
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000295{
296 return atomic64_inc_return(&fs_info->tree_mod_seq);
297}
298
299/*
Jan Schmidt097b8a72012-06-21 11:08:04 +0200300 * This adds a new blocker to the tree mod log's blocker list if the @elem
301 * passed does not already have a sequence number set. So when a caller expects
302 * to record tree modifications, it should ensure to set elem->seq to zero
303 * before calling btrfs_get_tree_mod_seq.
304 * Returns a fresh, unused tree log modification sequence number, even if no new
305 * blocker was added.
306 */
307u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info,
308 struct seq_list *elem)
309{
David Sterbab1a09f12018-03-05 15:43:41 +0100310 write_lock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200311 spin_lock(&fs_info->tree_mod_seq_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200312 if (!elem->seq) {
Josef Bacikfcebe452014-05-13 17:30:47 -0700313 elem->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200314 list_add_tail(&elem->list, &fs_info->tree_mod_seq_list);
315 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200316 spin_unlock(&fs_info->tree_mod_seq_lock);
David Sterbab1a09f12018-03-05 15:43:41 +0100317 write_unlock(&fs_info->tree_mod_log_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200318
Josef Bacikfcebe452014-05-13 17:30:47 -0700319 return elem->seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200320}
321
322void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info,
323 struct seq_list *elem)
324{
325 struct rb_root *tm_root;
326 struct rb_node *node;
327 struct rb_node *next;
328 struct seq_list *cur_elem;
329 struct tree_mod_elem *tm;
330 u64 min_seq = (u64)-1;
331 u64 seq_putting = elem->seq;
332
333 if (!seq_putting)
334 return;
335
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200336 spin_lock(&fs_info->tree_mod_seq_lock);
337 list_del(&elem->list);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200338 elem->seq = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200339
340 list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) {
Jan Schmidt097b8a72012-06-21 11:08:04 +0200341 if (cur_elem->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200342 if (seq_putting > cur_elem->seq) {
343 /*
344 * blocker with lower sequence number exists, we
345 * cannot remove anything from the log
346 */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200347 spin_unlock(&fs_info->tree_mod_seq_lock);
348 return;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200349 }
350 min_seq = cur_elem->seq;
351 }
352 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200353 spin_unlock(&fs_info->tree_mod_seq_lock);
354
355 /*
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200356 * anything that's lower than the lowest existing (read: blocked)
357 * sequence number can be removed from the tree.
358 */
David Sterbab1a09f12018-03-05 15:43:41 +0100359 write_lock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200360 tm_root = &fs_info->tree_mod_log;
361 for (node = rb_first(tm_root); node; node = next) {
362 next = rb_next(node);
Geliang Tang6b4df8b2016-12-19 22:53:41 +0800363 tm = rb_entry(node, struct tree_mod_elem, node);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200364 if (tm->seq > min_seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200365 continue;
366 rb_erase(node, tm_root);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200367 kfree(tm);
368 }
David Sterbab1a09f12018-03-05 15:43:41 +0100369 write_unlock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200370}
371
372/*
373 * key order of the log:
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530374 * node/leaf start address -> sequence
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200375 *
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530376 * The 'start address' is the logical address of the *new* root node
377 * for root replace operations, or the logical address of the affected
378 * block for all other operations.
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000379 *
David Sterbab1a09f12018-03-05 15:43:41 +0100380 * Note: must be called with write lock for fs_info::tree_mod_log_lock.
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200381 */
382static noinline int
383__tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm)
384{
385 struct rb_root *tm_root;
386 struct rb_node **new;
387 struct rb_node *parent = NULL;
388 struct tree_mod_elem *cur;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200389
Josef Bacikfcebe452014-05-13 17:30:47 -0700390 tm->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200391
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200392 tm_root = &fs_info->tree_mod_log;
393 new = &tm_root->rb_node;
394 while (*new) {
Geliang Tang6b4df8b2016-12-19 22:53:41 +0800395 cur = rb_entry(*new, struct tree_mod_elem, node);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200396 parent = *new;
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530397 if (cur->logical < tm->logical)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200398 new = &((*new)->rb_left);
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530399 else if (cur->logical > tm->logical)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200400 new = &((*new)->rb_right);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200401 else if (cur->seq < tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200402 new = &((*new)->rb_left);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200403 else if (cur->seq > tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200404 new = &((*new)->rb_right);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000405 else
406 return -EEXIST;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200407 }
408
409 rb_link_node(&tm->node, parent, new);
410 rb_insert_color(&tm->node, tm_root);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000411 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200412}
413
Jan Schmidt097b8a72012-06-21 11:08:04 +0200414/*
415 * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it
416 * returns zero with the tree_mod_log_lock acquired. The caller must hold
417 * this until all tree mod log insertions are recorded in the rb tree and then
David Sterbab1a09f12018-03-05 15:43:41 +0100418 * write unlock fs_info::tree_mod_log_lock.
Jan Schmidt097b8a72012-06-21 11:08:04 +0200419 */
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200420static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info,
421 struct extent_buffer *eb) {
422 smp_mb();
423 if (list_empty(&(fs_info)->tree_mod_seq_list))
424 return 1;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200425 if (eb && btrfs_header_level(eb) == 0)
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200426 return 1;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000427
David Sterbab1a09f12018-03-05 15:43:41 +0100428 write_lock(&fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000429 if (list_empty(&(fs_info)->tree_mod_seq_list)) {
David Sterbab1a09f12018-03-05 15:43:41 +0100430 write_unlock(&fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000431 return 1;
432 }
433
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200434 return 0;
435}
436
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000437/* Similar to tree_mod_dont_log, but doesn't acquire any locks. */
438static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info,
439 struct extent_buffer *eb)
440{
441 smp_mb();
442 if (list_empty(&(fs_info)->tree_mod_seq_list))
443 return 0;
444 if (eb && btrfs_header_level(eb) == 0)
445 return 0;
446
447 return 1;
448}
449
450static struct tree_mod_elem *
451alloc_tree_mod_elem(struct extent_buffer *eb, int slot,
452 enum mod_log_op op, gfp_t flags)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200453{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200454 struct tree_mod_elem *tm;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200455
Josef Bacikc8cc6342013-07-01 16:18:19 -0400456 tm = kzalloc(sizeof(*tm), flags);
457 if (!tm)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000458 return NULL;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200459
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530460 tm->logical = eb->start;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200461 if (op != MOD_LOG_KEY_ADD) {
462 btrfs_node_key(eb, &tm->key, slot);
463 tm->blockptr = btrfs_node_blockptr(eb, slot);
464 }
465 tm->op = op;
466 tm->slot = slot;
467 tm->generation = btrfs_node_ptr_generation(eb, slot);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000468 RB_CLEAR_NODE(&tm->node);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200469
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000470 return tm;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200471}
472
David Sterbae09c2ef2018-03-05 15:09:03 +0100473static noinline int tree_mod_log_insert_key(struct extent_buffer *eb, int slot,
474 enum mod_log_op op, gfp_t flags)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200475{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000476 struct tree_mod_elem *tm;
477 int ret;
478
David Sterbae09c2ef2018-03-05 15:09:03 +0100479 if (!tree_mod_need_log(eb->fs_info, eb))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200480 return 0;
481
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000482 tm = alloc_tree_mod_elem(eb, slot, op, flags);
483 if (!tm)
484 return -ENOMEM;
485
David Sterbae09c2ef2018-03-05 15:09:03 +0100486 if (tree_mod_dont_log(eb->fs_info, eb)) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000487 kfree(tm);
488 return 0;
489 }
490
David Sterbae09c2ef2018-03-05 15:09:03 +0100491 ret = __tree_mod_log_insert(eb->fs_info, tm);
David Sterbab1a09f12018-03-05 15:43:41 +0100492 write_unlock(&eb->fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000493 if (ret)
494 kfree(tm);
495
496 return ret;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200497}
498
David Sterba6074d452018-03-05 15:03:52 +0100499static noinline int tree_mod_log_insert_move(struct extent_buffer *eb,
500 int dst_slot, int src_slot, int nr_items)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200501{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000502 struct tree_mod_elem *tm = NULL;
503 struct tree_mod_elem **tm_list = NULL;
504 int ret = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200505 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000506 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200507
David Sterba6074d452018-03-05 15:03:52 +0100508 if (!tree_mod_need_log(eb->fs_info, eb))
Jan Schmidtf3956942012-05-31 15:02:32 +0200509 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200510
David Sterba176ef8f2017-03-28 14:35:01 +0200511 tm_list = kcalloc(nr_items, sizeof(struct tree_mod_elem *), GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000512 if (!tm_list)
513 return -ENOMEM;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200514
David Sterba176ef8f2017-03-28 14:35:01 +0200515 tm = kzalloc(sizeof(*tm), GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000516 if (!tm) {
517 ret = -ENOMEM;
518 goto free_tms;
519 }
Jan Schmidtf3956942012-05-31 15:02:32 +0200520
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530521 tm->logical = eb->start;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200522 tm->slot = src_slot;
523 tm->move.dst_slot = dst_slot;
524 tm->move.nr_items = nr_items;
525 tm->op = MOD_LOG_MOVE_KEYS;
526
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000527 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
528 tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot,
David Sterba176ef8f2017-03-28 14:35:01 +0200529 MOD_LOG_KEY_REMOVE_WHILE_MOVING, GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000530 if (!tm_list[i]) {
531 ret = -ENOMEM;
532 goto free_tms;
533 }
534 }
535
David Sterba6074d452018-03-05 15:03:52 +0100536 if (tree_mod_dont_log(eb->fs_info, eb))
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000537 goto free_tms;
538 locked = 1;
539
540 /*
541 * When we override something during the move, we log these removals.
542 * This can only happen when we move towards the beginning of the
543 * buffer, i.e. dst_slot < src_slot.
544 */
545 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
David Sterba6074d452018-03-05 15:03:52 +0100546 ret = __tree_mod_log_insert(eb->fs_info, tm_list[i]);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000547 if (ret)
548 goto free_tms;
549 }
550
David Sterba6074d452018-03-05 15:03:52 +0100551 ret = __tree_mod_log_insert(eb->fs_info, tm);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000552 if (ret)
553 goto free_tms;
David Sterbab1a09f12018-03-05 15:43:41 +0100554 write_unlock(&eb->fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000555 kfree(tm_list);
556
557 return 0;
558free_tms:
559 for (i = 0; i < nr_items; i++) {
560 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
David Sterba6074d452018-03-05 15:03:52 +0100561 rb_erase(&tm_list[i]->node, &eb->fs_info->tree_mod_log);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000562 kfree(tm_list[i]);
563 }
564 if (locked)
David Sterbab1a09f12018-03-05 15:43:41 +0100565 write_unlock(&eb->fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000566 kfree(tm_list);
567 kfree(tm);
568
569 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200570}
571
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000572static inline int
573__tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
574 struct tree_mod_elem **tm_list,
575 int nritems)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200576{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000577 int i, j;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200578 int ret;
579
Jan Schmidt097b8a72012-06-21 11:08:04 +0200580 for (i = nritems - 1; i >= 0; i--) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000581 ret = __tree_mod_log_insert(fs_info, tm_list[i]);
582 if (ret) {
583 for (j = nritems - 1; j > i; j--)
584 rb_erase(&tm_list[j]->node,
585 &fs_info->tree_mod_log);
586 return ret;
587 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200588 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000589
590 return 0;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200591}
592
David Sterba95b757c2018-03-05 15:22:30 +0100593static noinline int tree_mod_log_insert_root(struct extent_buffer *old_root,
594 struct extent_buffer *new_root, int log_removal)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200595{
David Sterba95b757c2018-03-05 15:22:30 +0100596 struct btrfs_fs_info *fs_info = old_root->fs_info;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000597 struct tree_mod_elem *tm = NULL;
598 struct tree_mod_elem **tm_list = NULL;
599 int nritems = 0;
600 int ret = 0;
601 int i;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200602
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000603 if (!tree_mod_need_log(fs_info, NULL))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200604 return 0;
605
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000606 if (log_removal && btrfs_header_level(old_root) > 0) {
607 nritems = btrfs_header_nritems(old_root);
David Sterba31e818f2015-02-20 18:00:26 +0100608 tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *),
David Sterbabcc8e072017-03-28 14:35:42 +0200609 GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000610 if (!tm_list) {
611 ret = -ENOMEM;
612 goto free_tms;
613 }
614 for (i = 0; i < nritems; i++) {
615 tm_list[i] = alloc_tree_mod_elem(old_root, i,
David Sterbabcc8e072017-03-28 14:35:42 +0200616 MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000617 if (!tm_list[i]) {
618 ret = -ENOMEM;
619 goto free_tms;
620 }
621 }
622 }
Jan Schmidtd9abbf12013-03-20 13:49:48 +0000623
David Sterbabcc8e072017-03-28 14:35:42 +0200624 tm = kzalloc(sizeof(*tm), GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000625 if (!tm) {
626 ret = -ENOMEM;
627 goto free_tms;
628 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200629
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530630 tm->logical = new_root->start;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200631 tm->old_root.logical = old_root->start;
632 tm->old_root.level = btrfs_header_level(old_root);
633 tm->generation = btrfs_header_generation(old_root);
634 tm->op = MOD_LOG_ROOT_REPLACE;
635
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000636 if (tree_mod_dont_log(fs_info, NULL))
637 goto free_tms;
638
639 if (tm_list)
640 ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
641 if (!ret)
642 ret = __tree_mod_log_insert(fs_info, tm);
643
David Sterbab1a09f12018-03-05 15:43:41 +0100644 write_unlock(&fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000645 if (ret)
646 goto free_tms;
647 kfree(tm_list);
648
649 return ret;
650
651free_tms:
652 if (tm_list) {
653 for (i = 0; i < nritems; i++)
654 kfree(tm_list[i]);
655 kfree(tm_list);
656 }
657 kfree(tm);
658
659 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200660}
661
662static struct tree_mod_elem *
663__tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq,
664 int smallest)
665{
666 struct rb_root *tm_root;
667 struct rb_node *node;
668 struct tree_mod_elem *cur = NULL;
669 struct tree_mod_elem *found = NULL;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200670
David Sterbab1a09f12018-03-05 15:43:41 +0100671 read_lock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200672 tm_root = &fs_info->tree_mod_log;
673 node = tm_root->rb_node;
674 while (node) {
Geliang Tang6b4df8b2016-12-19 22:53:41 +0800675 cur = rb_entry(node, struct tree_mod_elem, node);
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530676 if (cur->logical < start) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200677 node = node->rb_left;
Chandan Rajendra298cfd32016-01-21 15:55:59 +0530678 } else if (cur->logical > start) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200679 node = node->rb_right;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200680 } else if (cur->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200681 node = node->rb_left;
682 } else if (!smallest) {
683 /* we want the node with the highest seq */
684 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200685 BUG_ON(found->seq > cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200686 found = cur;
687 node = node->rb_left;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200688 } else if (cur->seq > min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200689 /* we want the node with the smallest seq */
690 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200691 BUG_ON(found->seq < cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200692 found = cur;
693 node = node->rb_right;
694 } else {
695 found = cur;
696 break;
697 }
698 }
David Sterbab1a09f12018-03-05 15:43:41 +0100699 read_unlock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200700
701 return found;
702}
703
704/*
705 * this returns the element from the log with the smallest time sequence
706 * value that's in the log (the oldest log item). any element with a time
707 * sequence lower than min_seq will be ignored.
708 */
709static struct tree_mod_elem *
710tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start,
711 u64 min_seq)
712{
713 return __tree_mod_log_search(fs_info, start, min_seq, 1);
714}
715
716/*
717 * this returns the element from the log with the largest time sequence
718 * value that's in the log (the most recent log item). any element with
719 * a time sequence lower than min_seq will be ignored.
720 */
721static struct tree_mod_elem *
722tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq)
723{
724 return __tree_mod_log_search(fs_info, start, min_seq, 0);
725}
726
David Sterbaed874f02019-03-20 14:22:04 +0100727static noinline int tree_mod_log_eb_copy(struct extent_buffer *dst,
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200728 struct extent_buffer *src, unsigned long dst_offset,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000729 unsigned long src_offset, int nr_items)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200730{
David Sterbaed874f02019-03-20 14:22:04 +0100731 struct btrfs_fs_info *fs_info = dst->fs_info;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000732 int ret = 0;
733 struct tree_mod_elem **tm_list = NULL;
734 struct tree_mod_elem **tm_list_add, **tm_list_rem;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200735 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000736 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200737
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000738 if (!tree_mod_need_log(fs_info, NULL))
739 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200740
Josef Bacikc8cc6342013-07-01 16:18:19 -0400741 if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000742 return 0;
743
David Sterba31e818f2015-02-20 18:00:26 +0100744 tm_list = kcalloc(nr_items * 2, sizeof(struct tree_mod_elem *),
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000745 GFP_NOFS);
746 if (!tm_list)
747 return -ENOMEM;
748
749 tm_list_add = tm_list;
750 tm_list_rem = tm_list + nr_items;
751 for (i = 0; i < nr_items; i++) {
752 tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset,
753 MOD_LOG_KEY_REMOVE, GFP_NOFS);
754 if (!tm_list_rem[i]) {
755 ret = -ENOMEM;
756 goto free_tms;
757 }
758
759 tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset,
760 MOD_LOG_KEY_ADD, GFP_NOFS);
761 if (!tm_list_add[i]) {
762 ret = -ENOMEM;
763 goto free_tms;
764 }
765 }
766
767 if (tree_mod_dont_log(fs_info, NULL))
768 goto free_tms;
769 locked = 1;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200770
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200771 for (i = 0; i < nr_items; i++) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000772 ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]);
773 if (ret)
774 goto free_tms;
775 ret = __tree_mod_log_insert(fs_info, tm_list_add[i]);
776 if (ret)
777 goto free_tms;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200778 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000779
David Sterbab1a09f12018-03-05 15:43:41 +0100780 write_unlock(&fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000781 kfree(tm_list);
782
783 return 0;
784
785free_tms:
786 for (i = 0; i < nr_items * 2; i++) {
787 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
788 rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
789 kfree(tm_list[i]);
790 }
791 if (locked)
David Sterbab1a09f12018-03-05 15:43:41 +0100792 write_unlock(&fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000793 kfree(tm_list);
794
795 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200796}
797
David Sterbadb7279a2018-03-05 15:14:25 +0100798static noinline int tree_mod_log_free_eb(struct extent_buffer *eb)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200799{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000800 struct tree_mod_elem **tm_list = NULL;
801 int nritems = 0;
802 int i;
803 int ret = 0;
804
805 if (btrfs_header_level(eb) == 0)
806 return 0;
807
David Sterbadb7279a2018-03-05 15:14:25 +0100808 if (!tree_mod_need_log(eb->fs_info, NULL))
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000809 return 0;
810
811 nritems = btrfs_header_nritems(eb);
David Sterba31e818f2015-02-20 18:00:26 +0100812 tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000813 if (!tm_list)
814 return -ENOMEM;
815
816 for (i = 0; i < nritems; i++) {
817 tm_list[i] = alloc_tree_mod_elem(eb, i,
818 MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS);
819 if (!tm_list[i]) {
820 ret = -ENOMEM;
821 goto free_tms;
822 }
823 }
824
David Sterbadb7279a2018-03-05 15:14:25 +0100825 if (tree_mod_dont_log(eb->fs_info, eb))
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000826 goto free_tms;
827
David Sterbadb7279a2018-03-05 15:14:25 +0100828 ret = __tree_mod_log_free_eb(eb->fs_info, tm_list, nritems);
David Sterbab1a09f12018-03-05 15:43:41 +0100829 write_unlock(&eb->fs_info->tree_mod_log_lock);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000830 if (ret)
831 goto free_tms;
832 kfree(tm_list);
833
834 return 0;
835
836free_tms:
837 for (i = 0; i < nritems; i++)
838 kfree(tm_list[i]);
839 kfree(tm_list);
840
841 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200842}
843
Chris Masond352ac62008-09-29 15:18:18 -0400844/*
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400845 * check if the tree block can be shared by multiple trees
846 */
847int btrfs_block_can_be_shared(struct btrfs_root *root,
848 struct extent_buffer *buf)
849{
850 /*
Nicholas D Steeves01327612016-05-19 21:18:45 -0400851 * Tree blocks not in reference counted trees and tree roots
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400852 * are never shared. If a block was allocated after the last
853 * snapshot and the block was not allocated by tree relocation,
854 * we know the block is not shared.
855 */
Miao Xie27cdeb72014-04-02 19:51:05 +0800856 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400857 buf != root->node && buf != root->commit_root &&
858 (btrfs_header_generation(buf) <=
859 btrfs_root_last_snapshot(&root->root_item) ||
860 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)))
861 return 1;
Nikolay Borisova79865c2018-06-21 09:45:00 +0300862
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400863 return 0;
864}
865
866static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
867 struct btrfs_root *root,
868 struct extent_buffer *buf,
Yan, Zhengf0486c62010-05-16 10:46:25 -0400869 struct extent_buffer *cow,
870 int *last_ref)
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400871{
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400872 struct btrfs_fs_info *fs_info = root->fs_info;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400873 u64 refs;
874 u64 owner;
875 u64 flags;
876 u64 new_flags = 0;
877 int ret;
878
879 /*
880 * Backrefs update rules:
881 *
882 * Always use full backrefs for extent pointers in tree block
883 * allocated by tree relocation.
884 *
885 * If a shared tree block is no longer referenced by its owner
886 * tree (btrfs_header_owner(buf) == root->root_key.objectid),
887 * use full backrefs for extent pointers in tree block.
888 *
889 * If a tree block is been relocating
890 * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID),
891 * use full backrefs for extent pointers in tree block.
892 * The reason for this is some operations (such as drop tree)
893 * are only allowed for blocks use full backrefs.
894 */
895
896 if (btrfs_block_can_be_shared(root, buf)) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400897 ret = btrfs_lookup_extent_info(trans, fs_info, buf->start,
Josef Bacik3173a182013-03-07 14:22:04 -0500898 btrfs_header_level(buf), 1,
899 &refs, &flags);
Mark Fashehbe1a5562011-08-08 13:20:18 -0700900 if (ret)
901 return ret;
Mark Fashehe5df9572011-08-29 14:17:04 -0700902 if (refs == 0) {
903 ret = -EROFS;
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400904 btrfs_handle_fs_error(fs_info, ret, NULL);
Mark Fashehe5df9572011-08-29 14:17:04 -0700905 return ret;
906 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400907 } else {
908 refs = 1;
909 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
910 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
911 flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
912 else
913 flags = 0;
914 }
915
916 owner = btrfs_header_owner(buf);
917 BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID &&
918 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
919
920 if (refs > 1) {
921 if ((owner == root->root_key.objectid ||
922 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) &&
923 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
Josef Bacike339a6b2014-07-02 10:54:25 -0700924 ret = btrfs_inc_ref(trans, root, buf, 1);
Jeff Mahoney692826b2017-11-21 13:58:49 -0500925 if (ret)
926 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400927
928 if (root->root_key.objectid ==
929 BTRFS_TREE_RELOC_OBJECTID) {
Josef Bacike339a6b2014-07-02 10:54:25 -0700930 ret = btrfs_dec_ref(trans, root, buf, 0);
Jeff Mahoney692826b2017-11-21 13:58:49 -0500931 if (ret)
932 return ret;
Josef Bacike339a6b2014-07-02 10:54:25 -0700933 ret = btrfs_inc_ref(trans, root, cow, 1);
Jeff Mahoney692826b2017-11-21 13:58:49 -0500934 if (ret)
935 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400936 }
937 new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
938 } else {
939
940 if (root->root_key.objectid ==
941 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -0700942 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400943 else
Josef Bacike339a6b2014-07-02 10:54:25 -0700944 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney692826b2017-11-21 13:58:49 -0500945 if (ret)
946 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400947 }
948 if (new_flags != 0) {
Josef Bacikb1c79e02013-05-09 13:49:30 -0400949 int level = btrfs_header_level(buf);
950
David Sterbaf5c8daa2019-03-20 11:43:36 +0100951 ret = btrfs_set_disk_extent_flags(trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400952 buf->start,
953 buf->len,
Josef Bacikb1c79e02013-05-09 13:49:30 -0400954 new_flags, level, 0);
Mark Fashehbe1a5562011-08-08 13:20:18 -0700955 if (ret)
956 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400957 }
958 } else {
959 if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
960 if (root->root_key.objectid ==
961 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -0700962 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400963 else
Josef Bacike339a6b2014-07-02 10:54:25 -0700964 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney692826b2017-11-21 13:58:49 -0500965 if (ret)
966 return ret;
Josef Bacike339a6b2014-07-02 10:54:25 -0700967 ret = btrfs_dec_ref(trans, root, buf, 1);
Jeff Mahoney692826b2017-11-21 13:58:49 -0500968 if (ret)
969 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400970 }
David Sterba6a884d7d2019-03-20 14:30:02 +0100971 btrfs_clean_tree_block(buf);
Yan, Zhengf0486c62010-05-16 10:46:25 -0400972 *last_ref = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400973 }
974 return 0;
975}
976
Filipe Mananaa6279472019-01-25 11:48:51 +0000977static struct extent_buffer *alloc_tree_block_no_bg_flush(
978 struct btrfs_trans_handle *trans,
979 struct btrfs_root *root,
980 u64 parent_start,
981 const struct btrfs_disk_key *disk_key,
982 int level,
983 u64 hint,
984 u64 empty_size)
985{
986 struct btrfs_fs_info *fs_info = root->fs_info;
987 struct extent_buffer *ret;
988
989 /*
990 * If we are COWing a node/leaf from the extent, chunk, device or free
991 * space trees, make sure that we do not finish block group creation of
992 * pending block groups. We do this to avoid a deadlock.
993 * COWing can result in allocation of a new chunk, and flushing pending
994 * block groups (btrfs_create_pending_block_groups()) can be triggered
995 * when finishing allocation of a new chunk. Creation of a pending block
996 * group modifies the extent, chunk, device and free space trees,
997 * therefore we could deadlock with ourselves since we are holding a
998 * lock on an extent buffer that btrfs_create_pending_block_groups() may
999 * try to COW later.
1000 * For similar reasons, we also need to delay flushing pending block
1001 * groups when splitting a leaf or node, from one of those trees, since
1002 * we are holding a write lock on it and its parent or when inserting a
1003 * new root node for one of those trees.
1004 */
1005 if (root == fs_info->extent_root ||
1006 root == fs_info->chunk_root ||
1007 root == fs_info->dev_root ||
1008 root == fs_info->free_space_root)
1009 trans->can_flush_pending_bgs = false;
1010
1011 ret = btrfs_alloc_tree_block(trans, root, parent_start,
1012 root->root_key.objectid, disk_key, level,
1013 hint, empty_size);
1014 trans->can_flush_pending_bgs = true;
1015
1016 return ret;
1017}
1018
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001019/*
Chris Masond3977122009-01-05 21:25:51 -05001020 * does the dirty work in cow of a single block. The parent block (if
1021 * supplied) is updated to point to the new cow copy. The new buffer is marked
1022 * dirty and returned locked. If you modify the block it needs to be marked
1023 * dirty again.
Chris Masond352ac62008-09-29 15:18:18 -04001024 *
1025 * search_start -- an allocation hint for the new block
1026 *
Chris Masond3977122009-01-05 21:25:51 -05001027 * empty_size -- a hint that you plan on doing more cow. This is the size in
1028 * bytes the allocator should try to find free next to the block it returns.
1029 * This is just a hint and may be ignored by the allocator.
Chris Masond352ac62008-09-29 15:18:18 -04001030 */
Chris Masond3977122009-01-05 21:25:51 -05001031static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001032 struct btrfs_root *root,
1033 struct extent_buffer *buf,
1034 struct extent_buffer *parent, int parent_slot,
1035 struct extent_buffer **cow_ret,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001036 u64 search_start, u64 empty_size)
Chris Mason6702ed42007-08-07 16:15:09 -04001037{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001038 struct btrfs_fs_info *fs_info = root->fs_info;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001039 struct btrfs_disk_key disk_key;
Chris Mason5f39d392007-10-15 16:14:19 -04001040 struct extent_buffer *cow;
Mark Fashehbe1a5562011-08-08 13:20:18 -07001041 int level, ret;
Yan, Zhengf0486c62010-05-16 10:46:25 -04001042 int last_ref = 0;
Chris Mason925baed2008-06-25 16:01:30 -04001043 int unlock_orig = 0;
Goldwyn Rodrigues0f5053e2016-09-22 14:11:34 -05001044 u64 parent_start = 0;
Chris Mason6702ed42007-08-07 16:15:09 -04001045
Chris Mason925baed2008-06-25 16:01:30 -04001046 if (*cow_ret == buf)
1047 unlock_orig = 1;
1048
Chris Masonb9447ef82009-03-09 11:45:38 -04001049 btrfs_assert_tree_locked(buf);
Chris Mason925baed2008-06-25 16:01:30 -04001050
Miao Xie27cdeb72014-04-02 19:51:05 +08001051 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001052 trans->transid != fs_info->running_transaction->transid);
Miao Xie27cdeb72014-04-02 19:51:05 +08001053 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1054 trans->transid != root->last_trans);
Chris Mason5f39d392007-10-15 16:14:19 -04001055
Chris Mason7bb86312007-12-11 09:25:06 -05001056 level = btrfs_header_level(buf);
Zheng Yan31840ae2008-09-23 13:14:14 -04001057
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001058 if (level == 0)
1059 btrfs_item_key(buf, &disk_key, 0);
1060 else
1061 btrfs_node_key(buf, &disk_key, 0);
1062
Goldwyn Rodrigues0f5053e2016-09-22 14:11:34 -05001063 if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent)
1064 parent_start = parent->start;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001065
Filipe Mananaa6279472019-01-25 11:48:51 +00001066 cow = alloc_tree_block_no_bg_flush(trans, root, parent_start, &disk_key,
1067 level, search_start, empty_size);
Chris Mason6702ed42007-08-07 16:15:09 -04001068 if (IS_ERR(cow))
1069 return PTR_ERR(cow);
1070
Chris Masonb4ce94d2009-02-04 09:25:08 -05001071 /* cow is set to blocking by btrfs_init_new_buffer */
1072
David Sterba58e80122016-11-08 18:30:31 +01001073 copy_extent_buffer_full(cow, buf);
Chris Masondb945352007-10-15 16:15:53 -04001074 btrfs_set_header_bytenr(cow, cow->start);
Chris Mason5f39d392007-10-15 16:14:19 -04001075 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001076 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
1077 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
1078 BTRFS_HEADER_FLAG_RELOC);
1079 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
1080 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
1081 else
1082 btrfs_set_header_owner(cow, root->root_key.objectid);
Chris Mason6702ed42007-08-07 16:15:09 -04001083
Nikolay Borisovde37aa52018-10-30 16:43:24 +02001084 write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid);
Yan Zheng2b820322008-11-17 21:11:30 -05001085
Mark Fashehbe1a5562011-08-08 13:20:18 -07001086 ret = update_ref_for_cow(trans, root, buf, cow, &last_ref);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001087 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04001088 btrfs_abort_transaction(trans, ret);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001089 return ret;
1090 }
Zheng Yan1a40e232008-09-26 10:09:34 -04001091
Miao Xie27cdeb72014-04-02 19:51:05 +08001092 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) {
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001093 ret = btrfs_reloc_cow_block(trans, root, buf, cow);
Zhaolei93314e32015-08-06 21:56:58 +08001094 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04001095 btrfs_abort_transaction(trans, ret);
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001096 return ret;
Zhaolei93314e32015-08-06 21:56:58 +08001097 }
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001098 }
Yan, Zheng3fd0a552010-05-16 10:49:59 -04001099
Chris Mason6702ed42007-08-07 16:15:09 -04001100 if (buf == root->node) {
Chris Mason925baed2008-06-25 16:01:30 -04001101 WARN_ON(parent && parent != buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001102 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
1103 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
1104 parent_start = buf->start;
Chris Mason925baed2008-06-25 16:01:30 -04001105
Chris Mason5f39d392007-10-15 16:14:19 -04001106 extent_buffer_get(cow);
David Sterbad9d19a02018-03-05 16:35:29 +01001107 ret = tree_mod_log_insert_root(root->node, cow, 1);
1108 BUG_ON(ret < 0);
Chris Mason240f62c2011-03-23 14:54:42 -04001109 rcu_assign_pointer(root->node, cow);
Chris Mason925baed2008-06-25 16:01:30 -04001110
Yan, Zhengf0486c62010-05-16 10:46:25 -04001111 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001112 last_ref);
Chris Mason5f39d392007-10-15 16:14:19 -04001113 free_extent_buffer(buf);
Chris Mason0b86a832008-03-24 15:01:56 -04001114 add_root_to_dirty_list(root);
Chris Mason6702ed42007-08-07 16:15:09 -04001115 } else {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001116 WARN_ON(trans->transid != btrfs_header_generation(parent));
David Sterbae09c2ef2018-03-05 15:09:03 +01001117 tree_mod_log_insert_key(parent, parent_slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04001118 MOD_LOG_KEY_REPLACE, GFP_NOFS);
Chris Mason5f39d392007-10-15 16:14:19 -04001119 btrfs_set_node_blockptr(parent, parent_slot,
Chris Masondb945352007-10-15 16:15:53 -04001120 cow->start);
Chris Mason74493f72007-12-11 09:25:06 -05001121 btrfs_set_node_ptr_generation(parent, parent_slot,
1122 trans->transid);
Chris Mason6702ed42007-08-07 16:15:09 -04001123 btrfs_mark_buffer_dirty(parent);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00001124 if (last_ref) {
David Sterbadb7279a2018-03-05 15:14:25 +01001125 ret = tree_mod_log_free_eb(buf);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00001126 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04001127 btrfs_abort_transaction(trans, ret);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00001128 return ret;
1129 }
1130 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04001131 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001132 last_ref);
Chris Mason6702ed42007-08-07 16:15:09 -04001133 }
Chris Mason925baed2008-06-25 16:01:30 -04001134 if (unlock_orig)
1135 btrfs_tree_unlock(buf);
Josef Bacik3083ee22012-03-09 16:01:49 -05001136 free_extent_buffer_stale(buf);
Chris Mason6702ed42007-08-07 16:15:09 -04001137 btrfs_mark_buffer_dirty(cow);
1138 *cow_ret = cow;
1139 return 0;
1140}
1141
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001142/*
1143 * returns the logical address of the oldest predecessor of the given root.
1144 * entries older than time_seq are ignored.
1145 */
David Sterbabcd24da2018-03-05 15:33:18 +01001146static struct tree_mod_elem *__tree_mod_log_oldest_root(
1147 struct extent_buffer *eb_root, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001148{
1149 struct tree_mod_elem *tm;
1150 struct tree_mod_elem *found = NULL;
Jan Schmidt30b04632013-04-13 13:19:54 +00001151 u64 root_logical = eb_root->start;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001152 int looped = 0;
1153
1154 if (!time_seq)
Stefan Behrens35a36212013-08-14 18:12:25 +02001155 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001156
1157 /*
Chandan Rajendra298cfd32016-01-21 15:55:59 +05301158 * the very last operation that's logged for a root is the
1159 * replacement operation (if it is replaced at all). this has
1160 * the logical address of the *new* root, making it the very
1161 * first operation that's logged for this root.
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001162 */
1163 while (1) {
David Sterbabcd24da2018-03-05 15:33:18 +01001164 tm = tree_mod_log_search_oldest(eb_root->fs_info, root_logical,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001165 time_seq);
1166 if (!looped && !tm)
Stefan Behrens35a36212013-08-14 18:12:25 +02001167 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001168 /*
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001169 * if there are no tree operation for the oldest root, we simply
1170 * return it. this should only happen if that (old) root is at
1171 * level 0.
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001172 */
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001173 if (!tm)
1174 break;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001175
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001176 /*
1177 * if there's an operation that's not a root replacement, we
1178 * found the oldest version of our root. normally, we'll find a
1179 * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here.
1180 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001181 if (tm->op != MOD_LOG_ROOT_REPLACE)
1182 break;
1183
1184 found = tm;
1185 root_logical = tm->old_root.logical;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001186 looped = 1;
1187 }
1188
Jan Schmidta95236d2012-06-05 16:41:24 +02001189 /* if there's no old root to return, return what we found instead */
1190 if (!found)
1191 found = tm;
1192
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001193 return found;
1194}
1195
1196/*
1197 * tm is a pointer to the first operation to rewind within eb. then, all
Nicholas D Steeves01327612016-05-19 21:18:45 -04001198 * previous operations will be rewound (until we reach something older than
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001199 * time_seq).
1200 */
1201static void
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001202__tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb,
1203 u64 time_seq, struct tree_mod_elem *first_tm)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001204{
1205 u32 n;
1206 struct rb_node *next;
1207 struct tree_mod_elem *tm = first_tm;
1208 unsigned long o_dst;
1209 unsigned long o_src;
1210 unsigned long p_size = sizeof(struct btrfs_key_ptr);
1211
1212 n = btrfs_header_nritems(eb);
David Sterbab1a09f12018-03-05 15:43:41 +01001213 read_lock(&fs_info->tree_mod_log_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +02001214 while (tm && tm->seq >= time_seq) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001215 /*
1216 * all the operations are recorded with the operator used for
1217 * the modification. as we're going backwards, we do the
1218 * opposite of each operation here.
1219 */
1220 switch (tm->op) {
1221 case MOD_LOG_KEY_REMOVE_WHILE_FREEING:
1222 BUG_ON(tm->slot < n);
Eric Sandeen1c697d42013-01-31 00:54:56 +00001223 /* Fallthrough */
Liu Bo95c80bb2012-10-19 09:50:52 +00001224 case MOD_LOG_KEY_REMOVE_WHILE_MOVING:
Chris Mason4c3e6962012-12-18 15:43:18 -05001225 case MOD_LOG_KEY_REMOVE:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001226 btrfs_set_node_key(eb, &tm->key, tm->slot);
1227 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1228 btrfs_set_node_ptr_generation(eb, tm->slot,
1229 tm->generation);
Chris Mason4c3e6962012-12-18 15:43:18 -05001230 n++;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001231 break;
1232 case MOD_LOG_KEY_REPLACE:
1233 BUG_ON(tm->slot >= n);
1234 btrfs_set_node_key(eb, &tm->key, tm->slot);
1235 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1236 btrfs_set_node_ptr_generation(eb, tm->slot,
1237 tm->generation);
1238 break;
1239 case MOD_LOG_KEY_ADD:
Jan Schmidt19956c72012-06-22 14:52:13 +02001240 /* if a move operation is needed it's in the log */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001241 n--;
1242 break;
1243 case MOD_LOG_MOVE_KEYS:
Jan Schmidtc3193102012-05-31 19:24:36 +02001244 o_dst = btrfs_node_key_ptr_offset(tm->slot);
1245 o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot);
1246 memmove_extent_buffer(eb, o_dst, o_src,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001247 tm->move.nr_items * p_size);
1248 break;
1249 case MOD_LOG_ROOT_REPLACE:
1250 /*
1251 * this operation is special. for roots, this must be
1252 * handled explicitly before rewinding.
1253 * for non-roots, this operation may exist if the node
1254 * was a root: root A -> child B; then A gets empty and
1255 * B is promoted to the new root. in the mod log, we'll
1256 * have a root-replace operation for B, a tree block
1257 * that is no root. we simply ignore that operation.
1258 */
1259 break;
1260 }
1261 next = rb_next(&tm->node);
1262 if (!next)
1263 break;
Geliang Tang6b4df8b2016-12-19 22:53:41 +08001264 tm = rb_entry(next, struct tree_mod_elem, node);
Chandan Rajendra298cfd32016-01-21 15:55:59 +05301265 if (tm->logical != first_tm->logical)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001266 break;
1267 }
David Sterbab1a09f12018-03-05 15:43:41 +01001268 read_unlock(&fs_info->tree_mod_log_lock);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001269 btrfs_set_header_nritems(eb, n);
1270}
1271
Jan Schmidt47fb0912013-04-13 13:19:55 +00001272/*
Nicholas D Steeves01327612016-05-19 21:18:45 -04001273 * Called with eb read locked. If the buffer cannot be rewound, the same buffer
Jan Schmidt47fb0912013-04-13 13:19:55 +00001274 * is returned. If rewind operations happen, a fresh buffer is returned. The
1275 * returned buffer is always read-locked. If the returned buffer is not the
1276 * input buffer, the lock on the input buffer is released and the input buffer
1277 * is freed (its refcount is decremented).
1278 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001279static struct extent_buffer *
Josef Bacik9ec72672013-08-07 16:57:23 -04001280tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path,
1281 struct extent_buffer *eb, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001282{
1283 struct extent_buffer *eb_rewin;
1284 struct tree_mod_elem *tm;
1285
1286 if (!time_seq)
1287 return eb;
1288
1289 if (btrfs_header_level(eb) == 0)
1290 return eb;
1291
1292 tm = tree_mod_log_search(fs_info, eb->start, time_seq);
1293 if (!tm)
1294 return eb;
1295
Josef Bacik9ec72672013-08-07 16:57:23 -04001296 btrfs_set_path_blocking(path);
David Sterba300aa892018-04-04 02:00:17 +02001297 btrfs_set_lock_blocking_read(eb);
Josef Bacik9ec72672013-08-07 16:57:23 -04001298
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001299 if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
1300 BUG_ON(tm->slot != 0);
Jeff Mahoneyda170662016-06-15 09:22:56 -04001301 eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001302 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001303 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001304 free_extent_buffer(eb);
1305 return NULL;
1306 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001307 btrfs_set_header_bytenr(eb_rewin, eb->start);
1308 btrfs_set_header_backref_rev(eb_rewin,
1309 btrfs_header_backref_rev(eb));
1310 btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb));
Jan Schmidtc3193102012-05-31 19:24:36 +02001311 btrfs_set_header_level(eb_rewin, btrfs_header_level(eb));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001312 } else {
1313 eb_rewin = btrfs_clone_extent_buffer(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001314 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001315 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001316 free_extent_buffer(eb);
1317 return NULL;
1318 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001319 }
1320
Josef Bacik9ec72672013-08-07 16:57:23 -04001321 btrfs_tree_read_unlock_blocking(eb);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001322 free_extent_buffer(eb);
1323
Jan Schmidt47fb0912013-04-13 13:19:55 +00001324 btrfs_tree_read_lock(eb_rewin);
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001325 __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm);
Jan Schmidt57911b82012-10-19 09:22:03 +02001326 WARN_ON(btrfs_header_nritems(eb_rewin) >
Jeff Mahoneyda170662016-06-15 09:22:56 -04001327 BTRFS_NODEPTRS_PER_BLOCK(fs_info));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001328
1329 return eb_rewin;
1330}
1331
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001332/*
1333 * get_old_root() rewinds the state of @root's root node to the given @time_seq
1334 * value. If there are no changes, the current root->root_node is returned. If
1335 * anything changed in between, there's a fresh buffer allocated on which the
1336 * rewind operations are done. In any case, the returned buffer is read locked.
1337 * Returns NULL on error (with no locks held).
1338 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001339static inline struct extent_buffer *
1340get_old_root(struct btrfs_root *root, u64 time_seq)
1341{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001342 struct btrfs_fs_info *fs_info = root->fs_info;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001343 struct tree_mod_elem *tm;
Jan Schmidt30b04632013-04-13 13:19:54 +00001344 struct extent_buffer *eb = NULL;
1345 struct extent_buffer *eb_root;
Liu Bo7bfdcf72012-10-25 07:30:19 -06001346 struct extent_buffer *old;
Jan Schmidta95236d2012-06-05 16:41:24 +02001347 struct tree_mod_root *old_root = NULL;
Chris Mason4325edd2012-06-15 20:02:02 -04001348 u64 old_generation = 0;
Jan Schmidta95236d2012-06-05 16:41:24 +02001349 u64 logical;
Qu Wenruo581c1762018-03-29 09:08:11 +08001350 int level;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001351
Jan Schmidt30b04632013-04-13 13:19:54 +00001352 eb_root = btrfs_read_lock_root_node(root);
David Sterbabcd24da2018-03-05 15:33:18 +01001353 tm = __tree_mod_log_oldest_root(eb_root, time_seq);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001354 if (!tm)
Jan Schmidt30b04632013-04-13 13:19:54 +00001355 return eb_root;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001356
Jan Schmidta95236d2012-06-05 16:41:24 +02001357 if (tm->op == MOD_LOG_ROOT_REPLACE) {
1358 old_root = &tm->old_root;
1359 old_generation = tm->generation;
1360 logical = old_root->logical;
Qu Wenruo581c1762018-03-29 09:08:11 +08001361 level = old_root->level;
Jan Schmidta95236d2012-06-05 16:41:24 +02001362 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001363 logical = eb_root->start;
Qu Wenruo581c1762018-03-29 09:08:11 +08001364 level = btrfs_header_level(eb_root);
Jan Schmidta95236d2012-06-05 16:41:24 +02001365 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001366
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001367 tm = tree_mod_log_search(fs_info, logical, time_seq);
Jan Schmidt834328a2012-10-23 11:27:33 +02001368 if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001369 btrfs_tree_read_unlock(eb_root);
1370 free_extent_buffer(eb_root);
Qu Wenruo581c1762018-03-29 09:08:11 +08001371 old = read_tree_block(fs_info, logical, 0, level, NULL);
Liu Bo64c043d2015-05-25 17:30:15 +08001372 if (WARN_ON(IS_ERR(old) || !extent_buffer_uptodate(old))) {
1373 if (!IS_ERR(old))
1374 free_extent_buffer(old);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001375 btrfs_warn(fs_info,
1376 "failed to read tree block %llu from get_old_root",
1377 logical);
Jan Schmidt834328a2012-10-23 11:27:33 +02001378 } else {
Liu Bo7bfdcf72012-10-25 07:30:19 -06001379 eb = btrfs_clone_extent_buffer(old);
1380 free_extent_buffer(old);
Jan Schmidt834328a2012-10-23 11:27:33 +02001381 }
1382 } else if (old_root) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001383 btrfs_tree_read_unlock(eb_root);
1384 free_extent_buffer(eb_root);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001385 eb = alloc_dummy_extent_buffer(fs_info, logical);
Jan Schmidt834328a2012-10-23 11:27:33 +02001386 } else {
David Sterba300aa892018-04-04 02:00:17 +02001387 btrfs_set_lock_blocking_read(eb_root);
Jan Schmidt30b04632013-04-13 13:19:54 +00001388 eb = btrfs_clone_extent_buffer(eb_root);
Josef Bacik9ec72672013-08-07 16:57:23 -04001389 btrfs_tree_read_unlock_blocking(eb_root);
Jan Schmidt30b04632013-04-13 13:19:54 +00001390 free_extent_buffer(eb_root);
Jan Schmidt834328a2012-10-23 11:27:33 +02001391 }
1392
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001393 if (!eb)
1394 return NULL;
1395 btrfs_tree_read_lock(eb);
Jan Schmidta95236d2012-06-05 16:41:24 +02001396 if (old_root) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001397 btrfs_set_header_bytenr(eb, eb->start);
1398 btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV);
Jan Schmidt30b04632013-04-13 13:19:54 +00001399 btrfs_set_header_owner(eb, btrfs_header_owner(eb_root));
Jan Schmidta95236d2012-06-05 16:41:24 +02001400 btrfs_set_header_level(eb, old_root->level);
1401 btrfs_set_header_generation(eb, old_generation);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001402 }
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001403 if (tm)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001404 __tree_mod_log_rewind(fs_info, eb, time_seq, tm);
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001405 else
1406 WARN_ON(btrfs_header_level(eb) != 0);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001407 WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(fs_info));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001408
1409 return eb;
1410}
1411
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001412int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq)
1413{
1414 struct tree_mod_elem *tm;
1415 int level;
Jan Schmidt30b04632013-04-13 13:19:54 +00001416 struct extent_buffer *eb_root = btrfs_root_node(root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001417
David Sterbabcd24da2018-03-05 15:33:18 +01001418 tm = __tree_mod_log_oldest_root(eb_root, time_seq);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001419 if (tm && tm->op == MOD_LOG_ROOT_REPLACE) {
1420 level = tm->old_root.level;
1421 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001422 level = btrfs_header_level(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001423 }
Jan Schmidt30b04632013-04-13 13:19:54 +00001424 free_extent_buffer(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001425
1426 return level;
1427}
1428
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001429static inline int should_cow_block(struct btrfs_trans_handle *trans,
1430 struct btrfs_root *root,
1431 struct extent_buffer *buf)
1432{
Jeff Mahoneyf5ee5c92016-06-21 09:52:41 -04001433 if (btrfs_is_testing(root->fs_info))
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04001434 return 0;
David Sterbafccb84c2014-09-29 23:53:21 +02001435
David Sterbad1980132018-03-16 02:39:40 +01001436 /* Ensure we can see the FORCE_COW bit */
1437 smp_mb__before_atomic();
Liu Bof1ebcc72011-11-14 20:48:06 -05001438
1439 /*
1440 * We do not need to cow a block if
1441 * 1) this block is not created or changed in this transaction;
1442 * 2) this block does not belong to TREE_RELOC tree;
1443 * 3) the root is not forced COW.
1444 *
1445 * What is forced COW:
Nicholas D Steeves01327612016-05-19 21:18:45 -04001446 * when we create snapshot during committing the transaction,
Andrea Gelmini52042d82018-11-28 12:05:13 +01001447 * after we've finished copying src root, we must COW the shared
Liu Bof1ebcc72011-11-14 20:48:06 -05001448 * block to ensure the metadata consistency.
1449 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001450 if (btrfs_header_generation(buf) == trans->transid &&
1451 !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) &&
1452 !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
Liu Bof1ebcc72011-11-14 20:48:06 -05001453 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) &&
Miao Xie27cdeb72014-04-02 19:51:05 +08001454 !test_bit(BTRFS_ROOT_FORCE_COW, &root->state))
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001455 return 0;
1456 return 1;
1457}
1458
Chris Masond352ac62008-09-29 15:18:18 -04001459/*
1460 * cows a single block, see __btrfs_cow_block for the real work.
Nicholas D Steeves01327612016-05-19 21:18:45 -04001461 * This version of it has extra checks so that a block isn't COWed more than
Chris Masond352ac62008-09-29 15:18:18 -04001462 * once per transaction, as long as it hasn't been written yet
1463 */
Chris Masond3977122009-01-05 21:25:51 -05001464noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001465 struct btrfs_root *root, struct extent_buffer *buf,
1466 struct extent_buffer *parent, int parent_slot,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001467 struct extent_buffer **cow_ret)
Chris Mason02217ed2007-03-02 16:08:05 -05001468{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001469 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason6702ed42007-08-07 16:15:09 -04001470 u64 search_start;
Chris Masonf510cfe2007-10-15 16:14:48 -04001471 int ret;
Chris Masondc17ff82008-01-08 15:46:30 -05001472
Josef Bacik83354f02018-11-30 11:52:13 -05001473 if (test_bit(BTRFS_ROOT_DELETING, &root->state))
1474 btrfs_err(fs_info,
1475 "COW'ing blocks on a fs root that's being dropped");
1476
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001477 if (trans->transaction != fs_info->running_transaction)
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001478 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001479 trans->transid,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001480 fs_info->running_transaction->transid);
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001481
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001482 if (trans->transid != fs_info->generation)
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001483 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001484 trans->transid, fs_info->generation);
Chris Masondc17ff82008-01-08 15:46:30 -05001485
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001486 if (!should_cow_block(trans, root, buf)) {
Jeff Mahoney64c12922016-06-08 00:36:38 -04001487 trans->dirty = true;
Chris Mason02217ed2007-03-02 16:08:05 -05001488 *cow_ret = buf;
1489 return 0;
1490 }
Chris Masonc4876852009-02-04 09:24:25 -05001491
Byongho Leeee221842015-12-15 01:42:10 +09001492 search_start = buf->start & ~((u64)SZ_1G - 1);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001493
1494 if (parent)
David Sterba8bead252018-04-04 02:03:48 +02001495 btrfs_set_lock_blocking_write(parent);
1496 btrfs_set_lock_blocking_write(buf);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001497
Qu Wenruof616f5c2019-01-23 15:15:17 +08001498 /*
1499 * Before CoWing this block for later modification, check if it's
1500 * the subtree root and do the delayed subtree trace if needed.
1501 *
1502 * Also We don't care about the error, as it's handled internally.
1503 */
1504 btrfs_qgroup_trace_subtree_after_cow(trans, root, buf);
Chris Masonf510cfe2007-10-15 16:14:48 -04001505 ret = __btrfs_cow_block(trans, root, buf, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001506 parent_slot, cow_ret, search_start, 0);
liubo1abe9b82011-03-24 11:18:59 +00001507
1508 trace_btrfs_cow_block(root, buf, *cow_ret);
1509
Chris Masonf510cfe2007-10-15 16:14:48 -04001510 return ret;
Chris Mason6702ed42007-08-07 16:15:09 -04001511}
1512
Chris Masond352ac62008-09-29 15:18:18 -04001513/*
1514 * helper function for defrag to decide if two blocks pointed to by a
1515 * node are actually close by
1516 */
Chris Mason6b800532007-10-15 16:17:34 -04001517static int close_blocks(u64 blocknr, u64 other, u32 blocksize)
Chris Mason6702ed42007-08-07 16:15:09 -04001518{
Chris Mason6b800532007-10-15 16:17:34 -04001519 if (blocknr < other && other - (blocknr + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001520 return 1;
Chris Mason6b800532007-10-15 16:17:34 -04001521 if (blocknr > other && blocknr - (other + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001522 return 1;
Chris Mason02217ed2007-03-02 16:08:05 -05001523 return 0;
1524}
1525
Chris Mason081e9572007-11-06 10:26:24 -05001526/*
1527 * compare two keys in a memcmp fashion
1528 */
Omar Sandoval310712b2017-01-17 23:24:37 -08001529static int comp_keys(const struct btrfs_disk_key *disk,
1530 const struct btrfs_key *k2)
Chris Mason081e9572007-11-06 10:26:24 -05001531{
1532 struct btrfs_key k1;
1533
1534 btrfs_disk_key_to_cpu(&k1, disk);
1535
Diego Calleja20736ab2009-07-24 11:06:52 -04001536 return btrfs_comp_cpu_keys(&k1, k2);
Chris Mason081e9572007-11-06 10:26:24 -05001537}
1538
Josef Bacikf3465ca2008-11-12 14:19:50 -05001539/*
1540 * same as comp_keys only with two btrfs_key's
1541 */
Omar Sandoval310712b2017-01-17 23:24:37 -08001542int btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2)
Josef Bacikf3465ca2008-11-12 14:19:50 -05001543{
1544 if (k1->objectid > k2->objectid)
1545 return 1;
1546 if (k1->objectid < k2->objectid)
1547 return -1;
1548 if (k1->type > k2->type)
1549 return 1;
1550 if (k1->type < k2->type)
1551 return -1;
1552 if (k1->offset > k2->offset)
1553 return 1;
1554 if (k1->offset < k2->offset)
1555 return -1;
1556 return 0;
1557}
Chris Mason081e9572007-11-06 10:26:24 -05001558
Chris Masond352ac62008-09-29 15:18:18 -04001559/*
1560 * this is used by the defrag code to go through all the
1561 * leaves pointed to by a node and reallocate them so that
1562 * disk order is close to key order
1563 */
Chris Mason6702ed42007-08-07 16:15:09 -04001564int btrfs_realloc_node(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001565 struct btrfs_root *root, struct extent_buffer *parent,
Eric Sandeende78b512013-01-31 18:21:12 +00001566 int start_slot, u64 *last_ret,
Chris Masona6b6e752007-10-15 16:22:39 -04001567 struct btrfs_key *progress)
Chris Mason6702ed42007-08-07 16:15:09 -04001568{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001569 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason6b800532007-10-15 16:17:34 -04001570 struct extent_buffer *cur;
Chris Mason6702ed42007-08-07 16:15:09 -04001571 u64 blocknr;
Chris Masonca7a79a2008-05-12 12:59:19 -04001572 u64 gen;
Chris Masone9d0b132007-08-10 14:06:19 -04001573 u64 search_start = *last_ret;
1574 u64 last_block = 0;
Chris Mason6702ed42007-08-07 16:15:09 -04001575 u64 other;
1576 u32 parent_nritems;
Chris Mason6702ed42007-08-07 16:15:09 -04001577 int end_slot;
1578 int i;
1579 int err = 0;
Chris Masonf2183bd2007-08-10 14:42:37 -04001580 int parent_level;
Chris Mason6b800532007-10-15 16:17:34 -04001581 int uptodate;
1582 u32 blocksize;
Chris Mason081e9572007-11-06 10:26:24 -05001583 int progress_passed = 0;
1584 struct btrfs_disk_key disk_key;
Chris Mason6702ed42007-08-07 16:15:09 -04001585
Chris Mason5708b952007-10-25 15:43:18 -04001586 parent_level = btrfs_header_level(parent);
Chris Mason5708b952007-10-25 15:43:18 -04001587
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001588 WARN_ON(trans->transaction != fs_info->running_transaction);
1589 WARN_ON(trans->transid != fs_info->generation);
Chris Mason86479a02007-09-10 19:58:16 -04001590
Chris Mason6b800532007-10-15 16:17:34 -04001591 parent_nritems = btrfs_header_nritems(parent);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001592 blocksize = fs_info->nodesize;
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001593 end_slot = parent_nritems - 1;
Chris Mason6702ed42007-08-07 16:15:09 -04001594
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001595 if (parent_nritems <= 1)
Chris Mason6702ed42007-08-07 16:15:09 -04001596 return 0;
1597
David Sterba8bead252018-04-04 02:03:48 +02001598 btrfs_set_lock_blocking_write(parent);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001599
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001600 for (i = start_slot; i <= end_slot; i++) {
Qu Wenruo581c1762018-03-29 09:08:11 +08001601 struct btrfs_key first_key;
Chris Mason6702ed42007-08-07 16:15:09 -04001602 int close = 1;
Chris Masona6b6e752007-10-15 16:22:39 -04001603
Chris Mason081e9572007-11-06 10:26:24 -05001604 btrfs_node_key(parent, &disk_key, i);
1605 if (!progress_passed && comp_keys(&disk_key, progress) < 0)
1606 continue;
1607
1608 progress_passed = 1;
Chris Mason6b800532007-10-15 16:17:34 -04001609 blocknr = btrfs_node_blockptr(parent, i);
Chris Masonca7a79a2008-05-12 12:59:19 -04001610 gen = btrfs_node_ptr_generation(parent, i);
Qu Wenruo581c1762018-03-29 09:08:11 +08001611 btrfs_node_key_to_cpu(parent, &first_key, i);
Chris Masone9d0b132007-08-10 14:06:19 -04001612 if (last_block == 0)
1613 last_block = blocknr;
Chris Mason5708b952007-10-25 15:43:18 -04001614
Chris Mason6702ed42007-08-07 16:15:09 -04001615 if (i > 0) {
Chris Mason6b800532007-10-15 16:17:34 -04001616 other = btrfs_node_blockptr(parent, i - 1);
1617 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001618 }
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001619 if (!close && i < end_slot) {
Chris Mason6b800532007-10-15 16:17:34 -04001620 other = btrfs_node_blockptr(parent, i + 1);
1621 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001622 }
Chris Masone9d0b132007-08-10 14:06:19 -04001623 if (close) {
1624 last_block = blocknr;
Chris Mason6702ed42007-08-07 16:15:09 -04001625 continue;
Chris Masone9d0b132007-08-10 14:06:19 -04001626 }
Chris Mason6702ed42007-08-07 16:15:09 -04001627
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001628 cur = find_extent_buffer(fs_info, blocknr);
Chris Mason6b800532007-10-15 16:17:34 -04001629 if (cur)
Chris Masonb9fab912012-05-06 07:23:47 -04001630 uptodate = btrfs_buffer_uptodate(cur, gen, 0);
Chris Mason6b800532007-10-15 16:17:34 -04001631 else
1632 uptodate = 0;
Chris Mason5708b952007-10-25 15:43:18 -04001633 if (!cur || !uptodate) {
Chris Mason6b800532007-10-15 16:17:34 -04001634 if (!cur) {
Qu Wenruo581c1762018-03-29 09:08:11 +08001635 cur = read_tree_block(fs_info, blocknr, gen,
1636 parent_level - 1,
1637 &first_key);
Liu Bo64c043d2015-05-25 17:30:15 +08001638 if (IS_ERR(cur)) {
1639 return PTR_ERR(cur);
1640 } else if (!extent_buffer_uptodate(cur)) {
Josef Bacik416bc652013-04-23 14:17:42 -04001641 free_extent_buffer(cur);
Tsutomu Itoh97d9a8a2011-03-24 06:33:21 +00001642 return -EIO;
Josef Bacik416bc652013-04-23 14:17:42 -04001643 }
Chris Mason6b800532007-10-15 16:17:34 -04001644 } else if (!uptodate) {
Qu Wenruo581c1762018-03-29 09:08:11 +08001645 err = btrfs_read_buffer(cur, gen,
1646 parent_level - 1,&first_key);
Tsutomu Itoh018642a2012-05-29 18:10:13 +09001647 if (err) {
1648 free_extent_buffer(cur);
1649 return err;
1650 }
Chris Masonf2183bd2007-08-10 14:42:37 -04001651 }
Chris Mason6702ed42007-08-07 16:15:09 -04001652 }
Chris Masone9d0b132007-08-10 14:06:19 -04001653 if (search_start == 0)
Chris Mason6b800532007-10-15 16:17:34 -04001654 search_start = last_block;
Chris Masone9d0b132007-08-10 14:06:19 -04001655
Chris Masone7a84562008-06-25 16:01:31 -04001656 btrfs_tree_lock(cur);
David Sterba8bead252018-04-04 02:03:48 +02001657 btrfs_set_lock_blocking_write(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001658 err = __btrfs_cow_block(trans, root, cur, parent, i,
Chris Masone7a84562008-06-25 16:01:31 -04001659 &cur, search_start,
Chris Mason6b800532007-10-15 16:17:34 -04001660 min(16 * blocksize,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001661 (end_slot - i) * blocksize));
Yan252c38f2007-08-29 09:11:44 -04001662 if (err) {
Chris Masone7a84562008-06-25 16:01:31 -04001663 btrfs_tree_unlock(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001664 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001665 break;
Yan252c38f2007-08-29 09:11:44 -04001666 }
Chris Masone7a84562008-06-25 16:01:31 -04001667 search_start = cur->start;
1668 last_block = cur->start;
Chris Masonf2183bd2007-08-10 14:42:37 -04001669 *last_ret = search_start;
Chris Masone7a84562008-06-25 16:01:31 -04001670 btrfs_tree_unlock(cur);
1671 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001672 }
1673 return err;
1674}
1675
Chris Mason74123bd2007-02-02 11:05:29 -05001676/*
Chris Mason5f39d392007-10-15 16:14:19 -04001677 * search for key in the extent_buffer. The items start at offset p,
1678 * and they are item_size apart. There are 'max' items in p.
1679 *
Chris Mason74123bd2007-02-02 11:05:29 -05001680 * the slot in the array is returned via slot, and it points to
1681 * the place where you would insert key if it is not found in
1682 * the array.
1683 *
1684 * slot may point to max if the key is bigger than all of the keys
1685 */
Chris Masone02119d2008-09-05 16:13:11 -04001686static noinline int generic_bin_search(struct extent_buffer *eb,
Omar Sandoval310712b2017-01-17 23:24:37 -08001687 unsigned long p, int item_size,
1688 const struct btrfs_key *key,
Chris Masone02119d2008-09-05 16:13:11 -04001689 int max, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001690{
1691 int low = 0;
1692 int high = max;
1693 int mid;
1694 int ret;
Chris Mason479965d2007-10-15 16:14:27 -04001695 struct btrfs_disk_key *tmp = NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001696 struct btrfs_disk_key unaligned;
1697 unsigned long offset;
Chris Mason5f39d392007-10-15 16:14:19 -04001698 char *kaddr = NULL;
1699 unsigned long map_start = 0;
1700 unsigned long map_len = 0;
Chris Mason479965d2007-10-15 16:14:27 -04001701 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05001702
Liu Bo5e24e9a2016-06-23 16:32:45 -07001703 if (low > high) {
1704 btrfs_err(eb->fs_info,
1705 "%s: low (%d) > high (%d) eb %llu owner %llu level %d",
1706 __func__, low, high, eb->start,
1707 btrfs_header_owner(eb), btrfs_header_level(eb));
1708 return -EINVAL;
1709 }
1710
Chris Masond3977122009-01-05 21:25:51 -05001711 while (low < high) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05001712 mid = (low + high) / 2;
Chris Mason5f39d392007-10-15 16:14:19 -04001713 offset = p + mid * item_size;
1714
Chris Masona6591712011-07-19 12:04:14 -04001715 if (!kaddr || offset < map_start ||
Chris Mason5f39d392007-10-15 16:14:19 -04001716 (offset + sizeof(struct btrfs_disk_key)) >
1717 map_start + map_len) {
Chris Mason934d3752008-12-08 16:43:10 -05001718
1719 err = map_private_extent_buffer(eb, offset,
Chris Mason479965d2007-10-15 16:14:27 -04001720 sizeof(struct btrfs_disk_key),
Chris Masona6591712011-07-19 12:04:14 -04001721 &kaddr, &map_start, &map_len);
Chris Mason5f39d392007-10-15 16:14:19 -04001722
Chris Mason479965d2007-10-15 16:14:27 -04001723 if (!err) {
1724 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1725 map_start);
Liu Bo415b35a2016-06-17 19:16:21 -07001726 } else if (err == 1) {
Chris Mason479965d2007-10-15 16:14:27 -04001727 read_extent_buffer(eb, &unaligned,
1728 offset, sizeof(unaligned));
1729 tmp = &unaligned;
Liu Bo415b35a2016-06-17 19:16:21 -07001730 } else {
1731 return err;
Chris Mason479965d2007-10-15 16:14:27 -04001732 }
1733
Chris Mason5f39d392007-10-15 16:14:19 -04001734 } else {
1735 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1736 map_start);
1737 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05001738 ret = comp_keys(tmp, key);
1739
1740 if (ret < 0)
1741 low = mid + 1;
1742 else if (ret > 0)
1743 high = mid;
1744 else {
1745 *slot = mid;
1746 return 0;
1747 }
1748 }
1749 *slot = low;
1750 return 1;
1751}
1752
Chris Mason97571fd2007-02-24 13:39:08 -05001753/*
1754 * simple bin_search frontend that does the right thing for
1755 * leaves vs nodes
1756 */
Nikolay Borisova74b35e2017-12-08 16:27:43 +02001757int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key,
1758 int level, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001759{
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001760 if (level == 0)
Chris Mason5f39d392007-10-15 16:14:19 -04001761 return generic_bin_search(eb,
1762 offsetof(struct btrfs_leaf, items),
Chris Mason0783fcf2007-03-12 20:12:07 -04001763 sizeof(struct btrfs_item),
Chris Mason5f39d392007-10-15 16:14:19 -04001764 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001765 slot);
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001766 else
Chris Mason5f39d392007-10-15 16:14:19 -04001767 return generic_bin_search(eb,
1768 offsetof(struct btrfs_node, ptrs),
Chris Mason123abc82007-03-14 14:14:43 -04001769 sizeof(struct btrfs_key_ptr),
Chris Mason5f39d392007-10-15 16:14:19 -04001770 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001771 slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001772}
1773
Yan, Zhengf0486c62010-05-16 10:46:25 -04001774static void root_add_used(struct btrfs_root *root, u32 size)
1775{
1776 spin_lock(&root->accounting_lock);
1777 btrfs_set_root_used(&root->root_item,
1778 btrfs_root_used(&root->root_item) + size);
1779 spin_unlock(&root->accounting_lock);
1780}
1781
1782static void root_sub_used(struct btrfs_root *root, u32 size)
1783{
1784 spin_lock(&root->accounting_lock);
1785 btrfs_set_root_used(&root->root_item,
1786 btrfs_root_used(&root->root_item) - size);
1787 spin_unlock(&root->accounting_lock);
1788}
1789
Chris Masond352ac62008-09-29 15:18:18 -04001790/* given a node and slot number, this reads the blocks it points to. The
1791 * extent buffer is returned with a reference taken (but unlocked).
Chris Masond352ac62008-09-29 15:18:18 -04001792 */
David Sterbad0d20b02019-03-20 14:54:01 +01001793static noinline struct extent_buffer *read_node_slot(
1794 struct extent_buffer *parent, int slot)
Chris Masonbb803952007-03-01 12:04:21 -05001795{
Chris Masonca7a79a2008-05-12 12:59:19 -04001796 int level = btrfs_header_level(parent);
Josef Bacik416bc652013-04-23 14:17:42 -04001797 struct extent_buffer *eb;
Qu Wenruo581c1762018-03-29 09:08:11 +08001798 struct btrfs_key first_key;
Josef Bacik416bc652013-04-23 14:17:42 -04001799
Liu Bofb770ae2016-07-05 12:10:14 -07001800 if (slot < 0 || slot >= btrfs_header_nritems(parent))
1801 return ERR_PTR(-ENOENT);
Chris Masonca7a79a2008-05-12 12:59:19 -04001802
1803 BUG_ON(level == 0);
1804
Qu Wenruo581c1762018-03-29 09:08:11 +08001805 btrfs_node_key_to_cpu(parent, &first_key, slot);
David Sterbad0d20b02019-03-20 14:54:01 +01001806 eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot),
Qu Wenruo581c1762018-03-29 09:08:11 +08001807 btrfs_node_ptr_generation(parent, slot),
1808 level - 1, &first_key);
Liu Bofb770ae2016-07-05 12:10:14 -07001809 if (!IS_ERR(eb) && !extent_buffer_uptodate(eb)) {
1810 free_extent_buffer(eb);
1811 eb = ERR_PTR(-EIO);
Josef Bacik416bc652013-04-23 14:17:42 -04001812 }
1813
1814 return eb;
Chris Masonbb803952007-03-01 12:04:21 -05001815}
1816
Chris Masond352ac62008-09-29 15:18:18 -04001817/*
1818 * node level balancing, used to make sure nodes are in proper order for
1819 * item deletion. We balance from the top down, so we have to make sure
1820 * that a deletion won't leave an node completely empty later on.
1821 */
Chris Masone02119d2008-09-05 16:13:11 -04001822static noinline int balance_level(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05001823 struct btrfs_root *root,
1824 struct btrfs_path *path, int level)
Chris Masonbb803952007-03-01 12:04:21 -05001825{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001826 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04001827 struct extent_buffer *right = NULL;
1828 struct extent_buffer *mid;
1829 struct extent_buffer *left = NULL;
1830 struct extent_buffer *parent = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05001831 int ret = 0;
1832 int wret;
1833 int pslot;
Chris Masonbb803952007-03-01 12:04:21 -05001834 int orig_slot = path->slots[level];
Chris Mason79f95c82007-03-01 15:16:26 -05001835 u64 orig_ptr;
Chris Masonbb803952007-03-01 12:04:21 -05001836
Liu Bo98e6b1e2018-09-12 06:06:23 +08001837 ASSERT(level > 0);
Chris Masonbb803952007-03-01 12:04:21 -05001838
Chris Mason5f39d392007-10-15 16:14:19 -04001839 mid = path->nodes[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05001840
Chris Masonbd681512011-07-16 15:23:14 -04001841 WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK &&
1842 path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING);
Chris Mason7bb86312007-12-11 09:25:06 -05001843 WARN_ON(btrfs_header_generation(mid) != trans->transid);
1844
Chris Mason1d4f8a02007-03-13 09:28:32 -04001845 orig_ptr = btrfs_node_blockptr(mid, orig_slot);
Chris Mason79f95c82007-03-01 15:16:26 -05001846
Li Zefana05a9bb2011-09-06 16:55:34 +08001847 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04001848 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08001849 pslot = path->slots[level + 1];
1850 }
Chris Masonbb803952007-03-01 12:04:21 -05001851
Chris Mason40689472007-03-17 14:29:23 -04001852 /*
1853 * deal with the case where there is only one pointer in the root
1854 * by promoting the node below to a root
1855 */
Chris Mason5f39d392007-10-15 16:14:19 -04001856 if (!parent) {
1857 struct extent_buffer *child;
Chris Masonbb803952007-03-01 12:04:21 -05001858
Chris Mason5f39d392007-10-15 16:14:19 -04001859 if (btrfs_header_nritems(mid) != 1)
Chris Masonbb803952007-03-01 12:04:21 -05001860 return 0;
1861
1862 /* promote the child to a root */
David Sterbad0d20b02019-03-20 14:54:01 +01001863 child = read_node_slot(mid, 0);
Liu Bofb770ae2016-07-05 12:10:14 -07001864 if (IS_ERR(child)) {
1865 ret = PTR_ERR(child);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001866 btrfs_handle_fs_error(fs_info, ret, NULL);
Mark Fasheh305a26a2011-09-01 11:27:57 -07001867 goto enospc;
1868 }
1869
Chris Mason925baed2008-06-25 16:01:30 -04001870 btrfs_tree_lock(child);
David Sterba8bead252018-04-04 02:03:48 +02001871 btrfs_set_lock_blocking_write(child);
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001872 ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001873 if (ret) {
1874 btrfs_tree_unlock(child);
1875 free_extent_buffer(child);
1876 goto enospc;
1877 }
Yan2f375ab2008-02-01 14:58:07 -05001878
David Sterbad9d19a02018-03-05 16:35:29 +01001879 ret = tree_mod_log_insert_root(root->node, child, 1);
1880 BUG_ON(ret < 0);
Chris Mason240f62c2011-03-23 14:54:42 -04001881 rcu_assign_pointer(root->node, child);
Chris Mason925baed2008-06-25 16:01:30 -04001882
Chris Mason0b86a832008-03-24 15:01:56 -04001883 add_root_to_dirty_list(root);
Chris Mason925baed2008-06-25 16:01:30 -04001884 btrfs_tree_unlock(child);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001885
Chris Mason925baed2008-06-25 16:01:30 -04001886 path->locks[level] = 0;
Chris Masonbb803952007-03-01 12:04:21 -05001887 path->nodes[level] = NULL;
David Sterba6a884d7d2019-03-20 14:30:02 +01001888 btrfs_clean_tree_block(mid);
Chris Mason925baed2008-06-25 16:01:30 -04001889 btrfs_tree_unlock(mid);
Chris Masonbb803952007-03-01 12:04:21 -05001890 /* once for the path */
Chris Mason5f39d392007-10-15 16:14:19 -04001891 free_extent_buffer(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001892
1893 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02001894 btrfs_free_tree_block(trans, root, mid, 0, 1);
Chris Masonbb803952007-03-01 12:04:21 -05001895 /* once for the root ptr */
Josef Bacik3083ee22012-03-09 16:01:49 -05001896 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001897 return 0;
Chris Masonbb803952007-03-01 12:04:21 -05001898 }
Chris Mason5f39d392007-10-15 16:14:19 -04001899 if (btrfs_header_nritems(mid) >
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001900 BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4)
Chris Masonbb803952007-03-01 12:04:21 -05001901 return 0;
1902
David Sterbad0d20b02019-03-20 14:54:01 +01001903 left = read_node_slot(parent, pslot - 1);
Liu Bofb770ae2016-07-05 12:10:14 -07001904 if (IS_ERR(left))
1905 left = NULL;
1906
Chris Mason5f39d392007-10-15 16:14:19 -04001907 if (left) {
Chris Mason925baed2008-06-25 16:01:30 -04001908 btrfs_tree_lock(left);
David Sterba8bead252018-04-04 02:03:48 +02001909 btrfs_set_lock_blocking_write(left);
Chris Mason5f39d392007-10-15 16:14:19 -04001910 wret = btrfs_cow_block(trans, root, left,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001911 parent, pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04001912 if (wret) {
1913 ret = wret;
1914 goto enospc;
1915 }
Chris Mason2cc58cf2007-08-27 16:49:44 -04001916 }
Liu Bofb770ae2016-07-05 12:10:14 -07001917
David Sterbad0d20b02019-03-20 14:54:01 +01001918 right = read_node_slot(parent, pslot + 1);
Liu Bofb770ae2016-07-05 12:10:14 -07001919 if (IS_ERR(right))
1920 right = NULL;
1921
Chris Mason5f39d392007-10-15 16:14:19 -04001922 if (right) {
Chris Mason925baed2008-06-25 16:01:30 -04001923 btrfs_tree_lock(right);
David Sterba8bead252018-04-04 02:03:48 +02001924 btrfs_set_lock_blocking_write(right);
Chris Mason5f39d392007-10-15 16:14:19 -04001925 wret = btrfs_cow_block(trans, root, right,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001926 parent, pslot + 1, &right);
Chris Mason2cc58cf2007-08-27 16:49:44 -04001927 if (wret) {
1928 ret = wret;
1929 goto enospc;
1930 }
1931 }
1932
1933 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04001934 if (left) {
1935 orig_slot += btrfs_header_nritems(left);
David Sterbad30a6682019-03-20 14:16:45 +01001936 wret = push_node_left(trans, left, mid, 1);
Chris Mason79f95c82007-03-01 15:16:26 -05001937 if (wret < 0)
1938 ret = wret;
Chris Masonbb803952007-03-01 12:04:21 -05001939 }
Chris Mason79f95c82007-03-01 15:16:26 -05001940
1941 /*
1942 * then try to empty the right most buffer into the middle
1943 */
Chris Mason5f39d392007-10-15 16:14:19 -04001944 if (right) {
David Sterbad30a6682019-03-20 14:16:45 +01001945 wret = push_node_left(trans, mid, right, 1);
Chris Mason54aa1f42007-06-22 14:16:25 -04001946 if (wret < 0 && wret != -ENOSPC)
Chris Mason79f95c82007-03-01 15:16:26 -05001947 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04001948 if (btrfs_header_nritems(right) == 0) {
David Sterba6a884d7d2019-03-20 14:30:02 +01001949 btrfs_clean_tree_block(right);
Chris Mason925baed2008-06-25 16:01:30 -04001950 btrfs_tree_unlock(right);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00001951 del_ptr(root, path, level + 1, pslot + 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001952 root_sub_used(root, right->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02001953 btrfs_free_tree_block(trans, root, right, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05001954 free_extent_buffer_stale(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001955 right = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05001956 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04001957 struct btrfs_disk_key right_key;
1958 btrfs_node_key(right, &right_key, 0);
David Sterba0e82bcf2018-03-05 16:16:54 +01001959 ret = tree_mod_log_insert_key(parent, pslot + 1,
1960 MOD_LOG_KEY_REPLACE, GFP_NOFS);
1961 BUG_ON(ret < 0);
Chris Mason5f39d392007-10-15 16:14:19 -04001962 btrfs_set_node_key(parent, &right_key, pslot + 1);
1963 btrfs_mark_buffer_dirty(parent);
Chris Masonbb803952007-03-01 12:04:21 -05001964 }
1965 }
Chris Mason5f39d392007-10-15 16:14:19 -04001966 if (btrfs_header_nritems(mid) == 1) {
Chris Mason79f95c82007-03-01 15:16:26 -05001967 /*
1968 * we're not allowed to leave a node with one item in the
1969 * tree during a delete. A deletion from lower in the tree
1970 * could try to delete the only pointer in this node.
1971 * So, pull some keys from the left.
1972 * There has to be a left pointer at this point because
1973 * otherwise we would have pulled some pointers from the
1974 * right
1975 */
Mark Fasheh305a26a2011-09-01 11:27:57 -07001976 if (!left) {
1977 ret = -EROFS;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001978 btrfs_handle_fs_error(fs_info, ret, NULL);
Mark Fasheh305a26a2011-09-01 11:27:57 -07001979 goto enospc;
1980 }
David Sterba55d32ed2019-03-20 14:18:06 +01001981 wret = balance_node_right(trans, mid, left);
Chris Mason54aa1f42007-06-22 14:16:25 -04001982 if (wret < 0) {
Chris Mason79f95c82007-03-01 15:16:26 -05001983 ret = wret;
Chris Mason54aa1f42007-06-22 14:16:25 -04001984 goto enospc;
1985 }
Chris Masonbce4eae2008-04-24 14:42:46 -04001986 if (wret == 1) {
David Sterbad30a6682019-03-20 14:16:45 +01001987 wret = push_node_left(trans, left, mid, 1);
Chris Masonbce4eae2008-04-24 14:42:46 -04001988 if (wret < 0)
1989 ret = wret;
1990 }
Chris Mason79f95c82007-03-01 15:16:26 -05001991 BUG_ON(wret == 1);
1992 }
Chris Mason5f39d392007-10-15 16:14:19 -04001993 if (btrfs_header_nritems(mid) == 0) {
David Sterba6a884d7d2019-03-20 14:30:02 +01001994 btrfs_clean_tree_block(mid);
Chris Mason925baed2008-06-25 16:01:30 -04001995 btrfs_tree_unlock(mid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00001996 del_ptr(root, path, level + 1, pslot);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001997 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02001998 btrfs_free_tree_block(trans, root, mid, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05001999 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002000 mid = NULL;
Chris Mason79f95c82007-03-01 15:16:26 -05002001 } else {
2002 /* update the parent key to reflect our changes */
Chris Mason5f39d392007-10-15 16:14:19 -04002003 struct btrfs_disk_key mid_key;
2004 btrfs_node_key(mid, &mid_key, 0);
David Sterba0e82bcf2018-03-05 16:16:54 +01002005 ret = tree_mod_log_insert_key(parent, pslot,
2006 MOD_LOG_KEY_REPLACE, GFP_NOFS);
2007 BUG_ON(ret < 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002008 btrfs_set_node_key(parent, &mid_key, pslot);
2009 btrfs_mark_buffer_dirty(parent);
Chris Mason79f95c82007-03-01 15:16:26 -05002010 }
Chris Masonbb803952007-03-01 12:04:21 -05002011
Chris Mason79f95c82007-03-01 15:16:26 -05002012 /* update the path */
Chris Mason5f39d392007-10-15 16:14:19 -04002013 if (left) {
2014 if (btrfs_header_nritems(left) > orig_slot) {
2015 extent_buffer_get(left);
Chris Mason925baed2008-06-25 16:01:30 -04002016 /* left was locked after cow */
Chris Mason5f39d392007-10-15 16:14:19 -04002017 path->nodes[level] = left;
Chris Masonbb803952007-03-01 12:04:21 -05002018 path->slots[level + 1] -= 1;
2019 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002020 if (mid) {
2021 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002022 free_extent_buffer(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002023 }
Chris Masonbb803952007-03-01 12:04:21 -05002024 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002025 orig_slot -= btrfs_header_nritems(left);
Chris Masonbb803952007-03-01 12:04:21 -05002026 path->slots[level] = orig_slot;
2027 }
2028 }
Chris Mason79f95c82007-03-01 15:16:26 -05002029 /* double check we haven't messed things up */
Chris Masone20d96d2007-03-22 12:13:20 -04002030 if (orig_ptr !=
Chris Mason5f39d392007-10-15 16:14:19 -04002031 btrfs_node_blockptr(path->nodes[level], path->slots[level]))
Chris Mason79f95c82007-03-01 15:16:26 -05002032 BUG();
Chris Mason54aa1f42007-06-22 14:16:25 -04002033enospc:
Chris Mason925baed2008-06-25 16:01:30 -04002034 if (right) {
2035 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002036 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04002037 }
2038 if (left) {
2039 if (path->nodes[level] != left)
2040 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002041 free_extent_buffer(left);
Chris Mason925baed2008-06-25 16:01:30 -04002042 }
Chris Masonbb803952007-03-01 12:04:21 -05002043 return ret;
2044}
2045
Chris Masond352ac62008-09-29 15:18:18 -04002046/* Node balancing for insertion. Here we only split or push nodes around
2047 * when they are completely full. This is also done top down, so we
2048 * have to be pessimistic.
2049 */
Chris Masond3977122009-01-05 21:25:51 -05002050static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05002051 struct btrfs_root *root,
2052 struct btrfs_path *path, int level)
Chris Masone66f7092007-04-20 13:16:02 -04002053{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002054 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04002055 struct extent_buffer *right = NULL;
2056 struct extent_buffer *mid;
2057 struct extent_buffer *left = NULL;
2058 struct extent_buffer *parent = NULL;
Chris Masone66f7092007-04-20 13:16:02 -04002059 int ret = 0;
2060 int wret;
2061 int pslot;
2062 int orig_slot = path->slots[level];
Chris Masone66f7092007-04-20 13:16:02 -04002063
2064 if (level == 0)
2065 return 1;
2066
Chris Mason5f39d392007-10-15 16:14:19 -04002067 mid = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05002068 WARN_ON(btrfs_header_generation(mid) != trans->transid);
Chris Masone66f7092007-04-20 13:16:02 -04002069
Li Zefana05a9bb2011-09-06 16:55:34 +08002070 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04002071 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08002072 pslot = path->slots[level + 1];
2073 }
Chris Masone66f7092007-04-20 13:16:02 -04002074
Chris Mason5f39d392007-10-15 16:14:19 -04002075 if (!parent)
Chris Masone66f7092007-04-20 13:16:02 -04002076 return 1;
Chris Masone66f7092007-04-20 13:16:02 -04002077
David Sterbad0d20b02019-03-20 14:54:01 +01002078 left = read_node_slot(parent, pslot - 1);
Liu Bofb770ae2016-07-05 12:10:14 -07002079 if (IS_ERR(left))
2080 left = NULL;
Chris Masone66f7092007-04-20 13:16:02 -04002081
2082 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04002083 if (left) {
Chris Masone66f7092007-04-20 13:16:02 -04002084 u32 left_nr;
Chris Mason925baed2008-06-25 16:01:30 -04002085
2086 btrfs_tree_lock(left);
David Sterba8bead252018-04-04 02:03:48 +02002087 btrfs_set_lock_blocking_write(left);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002088
Chris Mason5f39d392007-10-15 16:14:19 -04002089 left_nr = btrfs_header_nritems(left);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002090 if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) {
Chris Mason33ade1f2007-04-20 13:48:57 -04002091 wret = 1;
2092 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002093 ret = btrfs_cow_block(trans, root, left, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002094 pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04002095 if (ret)
2096 wret = 1;
2097 else {
David Sterbad30a6682019-03-20 14:16:45 +01002098 wret = push_node_left(trans, left, mid, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04002099 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002100 }
Chris Masone66f7092007-04-20 13:16:02 -04002101 if (wret < 0)
2102 ret = wret;
2103 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002104 struct btrfs_disk_key disk_key;
Chris Masone66f7092007-04-20 13:16:02 -04002105 orig_slot += left_nr;
Chris Mason5f39d392007-10-15 16:14:19 -04002106 btrfs_node_key(mid, &disk_key, 0);
David Sterba0e82bcf2018-03-05 16:16:54 +01002107 ret = tree_mod_log_insert_key(parent, pslot,
2108 MOD_LOG_KEY_REPLACE, GFP_NOFS);
2109 BUG_ON(ret < 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002110 btrfs_set_node_key(parent, &disk_key, pslot);
2111 btrfs_mark_buffer_dirty(parent);
2112 if (btrfs_header_nritems(left) > orig_slot) {
2113 path->nodes[level] = left;
Chris Masone66f7092007-04-20 13:16:02 -04002114 path->slots[level + 1] -= 1;
2115 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002116 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002117 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002118 } else {
2119 orig_slot -=
Chris Mason5f39d392007-10-15 16:14:19 -04002120 btrfs_header_nritems(left);
Chris Masone66f7092007-04-20 13:16:02 -04002121 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002122 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002123 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002124 }
Chris Masone66f7092007-04-20 13:16:02 -04002125 return 0;
2126 }
Chris Mason925baed2008-06-25 16:01:30 -04002127 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002128 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002129 }
David Sterbad0d20b02019-03-20 14:54:01 +01002130 right = read_node_slot(parent, pslot + 1);
Liu Bofb770ae2016-07-05 12:10:14 -07002131 if (IS_ERR(right))
2132 right = NULL;
Chris Masone66f7092007-04-20 13:16:02 -04002133
2134 /*
2135 * then try to empty the right most buffer into the middle
2136 */
Chris Mason5f39d392007-10-15 16:14:19 -04002137 if (right) {
Chris Mason33ade1f2007-04-20 13:48:57 -04002138 u32 right_nr;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002139
Chris Mason925baed2008-06-25 16:01:30 -04002140 btrfs_tree_lock(right);
David Sterba8bead252018-04-04 02:03:48 +02002141 btrfs_set_lock_blocking_write(right);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002142
Chris Mason5f39d392007-10-15 16:14:19 -04002143 right_nr = btrfs_header_nritems(right);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002144 if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) {
Chris Mason33ade1f2007-04-20 13:48:57 -04002145 wret = 1;
2146 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002147 ret = btrfs_cow_block(trans, root, right,
2148 parent, pslot + 1,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002149 &right);
Chris Mason54aa1f42007-06-22 14:16:25 -04002150 if (ret)
2151 wret = 1;
2152 else {
David Sterba55d32ed2019-03-20 14:18:06 +01002153 wret = balance_node_right(trans, right, mid);
Chris Mason54aa1f42007-06-22 14:16:25 -04002154 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002155 }
Chris Masone66f7092007-04-20 13:16:02 -04002156 if (wret < 0)
2157 ret = wret;
2158 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002159 struct btrfs_disk_key disk_key;
2160
2161 btrfs_node_key(right, &disk_key, 0);
David Sterba0e82bcf2018-03-05 16:16:54 +01002162 ret = tree_mod_log_insert_key(parent, pslot + 1,
2163 MOD_LOG_KEY_REPLACE, GFP_NOFS);
2164 BUG_ON(ret < 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002165 btrfs_set_node_key(parent, &disk_key, pslot + 1);
2166 btrfs_mark_buffer_dirty(parent);
2167
2168 if (btrfs_header_nritems(mid) <= orig_slot) {
2169 path->nodes[level] = right;
Chris Masone66f7092007-04-20 13:16:02 -04002170 path->slots[level + 1] += 1;
2171 path->slots[level] = orig_slot -
Chris Mason5f39d392007-10-15 16:14:19 -04002172 btrfs_header_nritems(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002173 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002174 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002175 } else {
Chris Mason925baed2008-06-25 16:01:30 -04002176 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002177 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002178 }
Chris Masone66f7092007-04-20 13:16:02 -04002179 return 0;
2180 }
Chris Mason925baed2008-06-25 16:01:30 -04002181 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002182 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002183 }
Chris Masone66f7092007-04-20 13:16:02 -04002184 return 1;
2185}
2186
Chris Mason74123bd2007-02-02 11:05:29 -05002187/*
Chris Masond352ac62008-09-29 15:18:18 -04002188 * readahead one full node of leaves, finding things that are close
2189 * to the block in 'slot', and triggering ra on them.
Chris Mason3c69fae2007-08-07 15:52:22 -04002190 */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002191static void reada_for_search(struct btrfs_fs_info *fs_info,
Chris Masonc8c42862009-04-03 10:14:18 -04002192 struct btrfs_path *path,
2193 int level, int slot, u64 objectid)
Chris Mason3c69fae2007-08-07 15:52:22 -04002194{
Chris Mason5f39d392007-10-15 16:14:19 -04002195 struct extent_buffer *node;
Chris Mason01f46652007-12-21 16:24:26 -05002196 struct btrfs_disk_key disk_key;
Chris Mason3c69fae2007-08-07 15:52:22 -04002197 u32 nritems;
Chris Mason3c69fae2007-08-07 15:52:22 -04002198 u64 search;
Chris Masona7175312009-01-22 09:23:10 -05002199 u64 target;
Chris Mason6b800532007-10-15 16:17:34 -04002200 u64 nread = 0;
Chris Mason5f39d392007-10-15 16:14:19 -04002201 struct extent_buffer *eb;
Chris Mason6b800532007-10-15 16:17:34 -04002202 u32 nr;
2203 u32 blocksize;
2204 u32 nscan = 0;
Chris Masondb945352007-10-15 16:15:53 -04002205
Chris Masona6b6e752007-10-15 16:22:39 -04002206 if (level != 1)
Chris Mason3c69fae2007-08-07 15:52:22 -04002207 return;
2208
Chris Mason6702ed42007-08-07 16:15:09 -04002209 if (!path->nodes[level])
2210 return;
2211
Chris Mason5f39d392007-10-15 16:14:19 -04002212 node = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04002213
Chris Mason3c69fae2007-08-07 15:52:22 -04002214 search = btrfs_node_blockptr(node, slot);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002215 blocksize = fs_info->nodesize;
2216 eb = find_extent_buffer(fs_info, search);
Chris Mason5f39d392007-10-15 16:14:19 -04002217 if (eb) {
2218 free_extent_buffer(eb);
Chris Mason3c69fae2007-08-07 15:52:22 -04002219 return;
2220 }
2221
Chris Masona7175312009-01-22 09:23:10 -05002222 target = search;
Chris Mason6b800532007-10-15 16:17:34 -04002223
Chris Mason5f39d392007-10-15 16:14:19 -04002224 nritems = btrfs_header_nritems(node);
Chris Mason6b800532007-10-15 16:17:34 -04002225 nr = slot;
Josef Bacik25b8b932011-06-08 14:36:54 -04002226
Chris Masond3977122009-01-05 21:25:51 -05002227 while (1) {
David Sterbae4058b52015-11-27 16:31:35 +01002228 if (path->reada == READA_BACK) {
Chris Mason6b800532007-10-15 16:17:34 -04002229 if (nr == 0)
2230 break;
2231 nr--;
David Sterbae4058b52015-11-27 16:31:35 +01002232 } else if (path->reada == READA_FORWARD) {
Chris Mason6b800532007-10-15 16:17:34 -04002233 nr++;
2234 if (nr >= nritems)
2235 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002236 }
David Sterbae4058b52015-11-27 16:31:35 +01002237 if (path->reada == READA_BACK && objectid) {
Chris Mason01f46652007-12-21 16:24:26 -05002238 btrfs_node_key(node, &disk_key, nr);
2239 if (btrfs_disk_key_objectid(&disk_key) != objectid)
2240 break;
2241 }
Chris Mason6b800532007-10-15 16:17:34 -04002242 search = btrfs_node_blockptr(node, nr);
Chris Masona7175312009-01-22 09:23:10 -05002243 if ((search <= target && target - search <= 65536) ||
2244 (search > target && search - target <= 65536)) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002245 readahead_tree_block(fs_info, search);
Chris Mason6b800532007-10-15 16:17:34 -04002246 nread += blocksize;
2247 }
2248 nscan++;
Chris Masona7175312009-01-22 09:23:10 -05002249 if ((nread > 65536 || nscan > 32))
Chris Mason6b800532007-10-15 16:17:34 -04002250 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002251 }
2252}
Chris Mason925baed2008-06-25 16:01:30 -04002253
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002254static noinline void reada_for_balance(struct btrfs_fs_info *fs_info,
Josef Bacik0b088512013-06-17 14:23:02 -04002255 struct btrfs_path *path, int level)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002256{
2257 int slot;
2258 int nritems;
2259 struct extent_buffer *parent;
2260 struct extent_buffer *eb;
2261 u64 gen;
2262 u64 block1 = 0;
2263 u64 block2 = 0;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002264
Chris Mason8c594ea2009-04-20 15:50:10 -04002265 parent = path->nodes[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002266 if (!parent)
Josef Bacik0b088512013-06-17 14:23:02 -04002267 return;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002268
2269 nritems = btrfs_header_nritems(parent);
Chris Mason8c594ea2009-04-20 15:50:10 -04002270 slot = path->slots[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002271
2272 if (slot > 0) {
2273 block1 = btrfs_node_blockptr(parent, slot - 1);
2274 gen = btrfs_node_ptr_generation(parent, slot - 1);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002275 eb = find_extent_buffer(fs_info, block1);
Chris Masonb9fab912012-05-06 07:23:47 -04002276 /*
2277 * if we get -eagain from btrfs_buffer_uptodate, we
2278 * don't want to return eagain here. That will loop
2279 * forever
2280 */
2281 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002282 block1 = 0;
2283 free_extent_buffer(eb);
2284 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002285 if (slot + 1 < nritems) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05002286 block2 = btrfs_node_blockptr(parent, slot + 1);
2287 gen = btrfs_node_ptr_generation(parent, slot + 1);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002288 eb = find_extent_buffer(fs_info, block2);
Chris Masonb9fab912012-05-06 07:23:47 -04002289 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002290 block2 = 0;
2291 free_extent_buffer(eb);
2292 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002293
Josef Bacik0b088512013-06-17 14:23:02 -04002294 if (block1)
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002295 readahead_tree_block(fs_info, block1);
Josef Bacik0b088512013-06-17 14:23:02 -04002296 if (block2)
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002297 readahead_tree_block(fs_info, block2);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002298}
2299
2300
2301/*
Chris Masond3977122009-01-05 21:25:51 -05002302 * when we walk down the tree, it is usually safe to unlock the higher layers
2303 * in the tree. The exceptions are when our path goes through slot 0, because
2304 * operations on the tree might require changing key pointers higher up in the
2305 * tree.
Chris Masond352ac62008-09-29 15:18:18 -04002306 *
Chris Masond3977122009-01-05 21:25:51 -05002307 * callers might also have set path->keep_locks, which tells this code to keep
2308 * the lock if the path points to the last slot in the block. This is part of
2309 * walking through the tree, and selecting the next slot in the higher block.
Chris Masond352ac62008-09-29 15:18:18 -04002310 *
Chris Masond3977122009-01-05 21:25:51 -05002311 * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so
2312 * if lowest_unlock is 1, level 0 won't be unlocked
Chris Masond352ac62008-09-29 15:18:18 -04002313 */
Chris Masone02119d2008-09-05 16:13:11 -04002314static noinline void unlock_up(struct btrfs_path *path, int level,
Chris Masonf7c79f32012-03-19 15:54:38 -04002315 int lowest_unlock, int min_write_lock_level,
2316 int *write_lock_level)
Chris Mason925baed2008-06-25 16:01:30 -04002317{
2318 int i;
2319 int skip_level = level;
Chris Mason051e1b92008-06-25 16:01:30 -04002320 int no_skips = 0;
Chris Mason925baed2008-06-25 16:01:30 -04002321 struct extent_buffer *t;
2322
2323 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2324 if (!path->nodes[i])
2325 break;
2326 if (!path->locks[i])
2327 break;
Chris Mason051e1b92008-06-25 16:01:30 -04002328 if (!no_skips && path->slots[i] == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002329 skip_level = i + 1;
2330 continue;
2331 }
Chris Mason051e1b92008-06-25 16:01:30 -04002332 if (!no_skips && path->keep_locks) {
Chris Mason925baed2008-06-25 16:01:30 -04002333 u32 nritems;
2334 t = path->nodes[i];
2335 nritems = btrfs_header_nritems(t);
Chris Mason051e1b92008-06-25 16:01:30 -04002336 if (nritems < 1 || path->slots[i] >= nritems - 1) {
Chris Mason925baed2008-06-25 16:01:30 -04002337 skip_level = i + 1;
2338 continue;
2339 }
2340 }
Chris Mason051e1b92008-06-25 16:01:30 -04002341 if (skip_level < i && i >= lowest_unlock)
2342 no_skips = 1;
2343
Chris Mason925baed2008-06-25 16:01:30 -04002344 t = path->nodes[i];
Liu Bod80bb3f2018-05-18 11:00:24 +08002345 if (i >= lowest_unlock && i > skip_level) {
Chris Masonbd681512011-07-16 15:23:14 -04002346 btrfs_tree_unlock_rw(t, path->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -04002347 path->locks[i] = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002348 if (write_lock_level &&
2349 i > min_write_lock_level &&
2350 i <= *write_lock_level) {
2351 *write_lock_level = i - 1;
2352 }
Chris Mason925baed2008-06-25 16:01:30 -04002353 }
2354 }
2355}
2356
Chris Mason3c69fae2007-08-07 15:52:22 -04002357/*
Chris Masonb4ce94d2009-02-04 09:25:08 -05002358 * This releases any locks held in the path starting at level and
2359 * going all the way up to the root.
2360 *
2361 * btrfs_search_slot will keep the lock held on higher nodes in a few
2362 * corner cases, such as COW of the block at slot zero in the node. This
2363 * ignores those rules, and it should only be called when there are no
2364 * more updates to be done higher up in the tree.
2365 */
2366noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level)
2367{
2368 int i;
2369
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002370 if (path->keep_locks)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002371 return;
2372
2373 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2374 if (!path->nodes[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002375 continue;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002376 if (!path->locks[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002377 continue;
Chris Masonbd681512011-07-16 15:23:14 -04002378 btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002379 path->locks[i] = 0;
2380 }
2381}
2382
2383/*
Chris Masonc8c42862009-04-03 10:14:18 -04002384 * helper function for btrfs_search_slot. The goal is to find a block
2385 * in cache without setting the path to blocking. If we find the block
2386 * we return zero and the path is unchanged.
2387 *
2388 * If we can't find the block, we set the path blocking and do some
2389 * reada. -EAGAIN is returned and the search must be repeated.
2390 */
2391static int
Liu Bod07b8522017-01-30 12:23:42 -08002392read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
2393 struct extent_buffer **eb_ret, int level, int slot,
David Sterbacda79c52017-02-10 18:44:32 +01002394 const struct btrfs_key *key)
Chris Masonc8c42862009-04-03 10:14:18 -04002395{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002396 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Masonc8c42862009-04-03 10:14:18 -04002397 u64 blocknr;
2398 u64 gen;
Chris Masonc8c42862009-04-03 10:14:18 -04002399 struct extent_buffer *b = *eb_ret;
2400 struct extent_buffer *tmp;
Qu Wenruo581c1762018-03-29 09:08:11 +08002401 struct btrfs_key first_key;
Chris Mason76a05b32009-05-14 13:24:30 -04002402 int ret;
Qu Wenruo581c1762018-03-29 09:08:11 +08002403 int parent_level;
Chris Masonc8c42862009-04-03 10:14:18 -04002404
2405 blocknr = btrfs_node_blockptr(b, slot);
2406 gen = btrfs_node_ptr_generation(b, slot);
Qu Wenruo581c1762018-03-29 09:08:11 +08002407 parent_level = btrfs_header_level(b);
2408 btrfs_node_key_to_cpu(b, &first_key, slot);
Chris Masonc8c42862009-04-03 10:14:18 -04002409
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002410 tmp = find_extent_buffer(fs_info, blocknr);
Chris Masoncb449212010-10-24 11:01:27 -04002411 if (tmp) {
Chris Masonb9fab912012-05-06 07:23:47 -04002412 /* first we do an atomic uptodate check */
Josef Bacikbdf7c002013-06-17 13:44:48 -04002413 if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) {
Qu Wenruo448de472019-03-12 17:10:40 +08002414 /*
2415 * Do extra check for first_key, eb can be stale due to
2416 * being cached, read from scrub, or have multiple
2417 * parents (shared tree blocks).
2418 */
David Sterbae064d5e2019-03-20 14:58:13 +01002419 if (btrfs_verify_level_key(tmp,
Qu Wenruo448de472019-03-12 17:10:40 +08002420 parent_level - 1, &first_key, gen)) {
2421 free_extent_buffer(tmp);
2422 return -EUCLEAN;
2423 }
Josef Bacikbdf7c002013-06-17 13:44:48 -04002424 *eb_ret = tmp;
2425 return 0;
Chris Masoncb449212010-10-24 11:01:27 -04002426 }
Josef Bacikbdf7c002013-06-17 13:44:48 -04002427
2428 /* the pages were up to date, but we failed
2429 * the generation number check. Do a full
2430 * read for the generation number that is correct.
2431 * We must do this without dropping locks so
2432 * we can trust our generation number
2433 */
2434 btrfs_set_path_blocking(p);
2435
2436 /* now we're allowed to do a blocking uptodate check */
Qu Wenruo581c1762018-03-29 09:08:11 +08002437 ret = btrfs_read_buffer(tmp, gen, parent_level - 1, &first_key);
Josef Bacikbdf7c002013-06-17 13:44:48 -04002438 if (!ret) {
2439 *eb_ret = tmp;
2440 return 0;
2441 }
2442 free_extent_buffer(tmp);
2443 btrfs_release_path(p);
2444 return -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002445 }
2446
2447 /*
2448 * reduce lock contention at high levels
2449 * of the btree by dropping locks before
Chris Mason76a05b32009-05-14 13:24:30 -04002450 * we read. Don't release the lock on the current
2451 * level because we need to walk this node to figure
2452 * out which blocks to read.
Chris Masonc8c42862009-04-03 10:14:18 -04002453 */
Chris Mason8c594ea2009-04-20 15:50:10 -04002454 btrfs_unlock_up_safe(p, level + 1);
2455 btrfs_set_path_blocking(p);
2456
David Sterbae4058b52015-11-27 16:31:35 +01002457 if (p->reada != READA_NONE)
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002458 reada_for_search(fs_info, p, level, slot, key->objectid);
Chris Masonc8c42862009-04-03 10:14:18 -04002459
Chris Mason76a05b32009-05-14 13:24:30 -04002460 ret = -EAGAIN;
Liu Bo02a33072018-05-16 01:37:36 +08002461 tmp = read_tree_block(fs_info, blocknr, gen, parent_level - 1,
Qu Wenruo581c1762018-03-29 09:08:11 +08002462 &first_key);
Liu Bo64c043d2015-05-25 17:30:15 +08002463 if (!IS_ERR(tmp)) {
Chris Mason76a05b32009-05-14 13:24:30 -04002464 /*
2465 * If the read above didn't mark this buffer up to date,
2466 * it will never end up being up to date. Set ret to EIO now
2467 * and give up so that our caller doesn't loop forever
2468 * on our EAGAINs.
2469 */
Liu Boe6a1d6f2018-05-18 11:00:20 +08002470 if (!extent_buffer_uptodate(tmp))
Chris Mason76a05b32009-05-14 13:24:30 -04002471 ret = -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002472 free_extent_buffer(tmp);
Liu Boc871b0f2016-06-06 12:01:23 -07002473 } else {
2474 ret = PTR_ERR(tmp);
Chris Mason76a05b32009-05-14 13:24:30 -04002475 }
Liu Bo02a33072018-05-16 01:37:36 +08002476
2477 btrfs_release_path(p);
Chris Mason76a05b32009-05-14 13:24:30 -04002478 return ret;
Chris Masonc8c42862009-04-03 10:14:18 -04002479}
2480
2481/*
2482 * helper function for btrfs_search_slot. This does all of the checks
2483 * for node-level blocks and does any balancing required based on
2484 * the ins_len.
2485 *
2486 * If no extra work was required, zero is returned. If we had to
2487 * drop the path, -EAGAIN is returned and btrfs_search_slot must
2488 * start over
2489 */
2490static int
2491setup_nodes_for_search(struct btrfs_trans_handle *trans,
2492 struct btrfs_root *root, struct btrfs_path *p,
Chris Masonbd681512011-07-16 15:23:14 -04002493 struct extent_buffer *b, int level, int ins_len,
2494 int *write_lock_level)
Chris Masonc8c42862009-04-03 10:14:18 -04002495{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002496 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Masonc8c42862009-04-03 10:14:18 -04002497 int ret;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002498
Chris Masonc8c42862009-04-03 10:14:18 -04002499 if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >=
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002500 BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) {
Chris Masonc8c42862009-04-03 10:14:18 -04002501 int sret;
2502
Chris Masonbd681512011-07-16 15:23:14 -04002503 if (*write_lock_level < level + 1) {
2504 *write_lock_level = level + 1;
2505 btrfs_release_path(p);
2506 goto again;
2507 }
2508
Chris Masonc8c42862009-04-03 10:14:18 -04002509 btrfs_set_path_blocking(p);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002510 reada_for_balance(fs_info, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002511 sret = split_node(trans, root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002512
2513 BUG_ON(sret > 0);
2514 if (sret) {
2515 ret = sret;
2516 goto done;
2517 }
2518 b = p->nodes[level];
2519 } else if (ins_len < 0 && btrfs_header_nritems(b) <
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002520 BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) {
Chris Masonc8c42862009-04-03 10:14:18 -04002521 int sret;
2522
Chris Masonbd681512011-07-16 15:23:14 -04002523 if (*write_lock_level < level + 1) {
2524 *write_lock_level = level + 1;
2525 btrfs_release_path(p);
2526 goto again;
2527 }
2528
Chris Masonc8c42862009-04-03 10:14:18 -04002529 btrfs_set_path_blocking(p);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002530 reada_for_balance(fs_info, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002531 sret = balance_level(trans, root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002532
2533 if (sret) {
2534 ret = sret;
2535 goto done;
2536 }
2537 b = p->nodes[level];
2538 if (!b) {
David Sterbab3b4aa72011-04-21 01:20:15 +02002539 btrfs_release_path(p);
Chris Masonc8c42862009-04-03 10:14:18 -04002540 goto again;
2541 }
2542 BUG_ON(btrfs_header_nritems(b) == 1);
2543 }
2544 return 0;
2545
2546again:
2547 ret = -EAGAIN;
2548done:
2549 return ret;
2550}
2551
Omar Sandoval310712b2017-01-17 23:24:37 -08002552static int key_search(struct extent_buffer *b, const struct btrfs_key *key,
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002553 int level, int *prev_cmp, int *slot)
2554{
2555 if (*prev_cmp != 0) {
Nikolay Borisova74b35e2017-12-08 16:27:43 +02002556 *prev_cmp = btrfs_bin_search(b, key, level, slot);
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002557 return *prev_cmp;
2558 }
2559
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002560 *slot = 0;
2561
2562 return 0;
2563}
2564
David Sterba381cf652015-01-02 18:45:16 +01002565int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path,
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002566 u64 iobjectid, u64 ioff, u8 key_type,
2567 struct btrfs_key *found_key)
2568{
2569 int ret;
2570 struct btrfs_key key;
2571 struct extent_buffer *eb;
David Sterba381cf652015-01-02 18:45:16 +01002572
2573 ASSERT(path);
David Sterba1d4c08e2015-01-02 19:36:14 +01002574 ASSERT(found_key);
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002575
2576 key.type = key_type;
2577 key.objectid = iobjectid;
2578 key.offset = ioff;
2579
2580 ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0);
David Sterba1d4c08e2015-01-02 19:36:14 +01002581 if (ret < 0)
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002582 return ret;
2583
2584 eb = path->nodes[0];
2585 if (ret && path->slots[0] >= btrfs_header_nritems(eb)) {
2586 ret = btrfs_next_leaf(fs_root, path);
2587 if (ret)
2588 return ret;
2589 eb = path->nodes[0];
2590 }
2591
2592 btrfs_item_key_to_cpu(eb, found_key, path->slots[0]);
2593 if (found_key->type != key.type ||
2594 found_key->objectid != key.objectid)
2595 return 1;
2596
2597 return 0;
2598}
2599
Liu Bo1fc28d82018-05-18 11:00:21 +08002600static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root,
2601 struct btrfs_path *p,
2602 int write_lock_level)
2603{
2604 struct btrfs_fs_info *fs_info = root->fs_info;
2605 struct extent_buffer *b;
2606 int root_lock;
2607 int level = 0;
2608
2609 /* We try very hard to do read locks on the root */
2610 root_lock = BTRFS_READ_LOCK;
2611
2612 if (p->search_commit_root) {
Filipe Mananabe6821f2018-12-11 10:19:45 +00002613 /*
2614 * The commit roots are read only so we always do read locks,
2615 * and we always must hold the commit_root_sem when doing
2616 * searches on them, the only exception is send where we don't
2617 * want to block transaction commits for a long time, so
2618 * we need to clone the commit root in order to avoid races
2619 * with transaction commits that create a snapshot of one of
2620 * the roots used by a send operation.
2621 */
2622 if (p->need_commit_sem) {
Liu Bo1fc28d82018-05-18 11:00:21 +08002623 down_read(&fs_info->commit_root_sem);
Filipe Mananabe6821f2018-12-11 10:19:45 +00002624 b = btrfs_clone_extent_buffer(root->commit_root);
Liu Bo1fc28d82018-05-18 11:00:21 +08002625 up_read(&fs_info->commit_root_sem);
Filipe Mananabe6821f2018-12-11 10:19:45 +00002626 if (!b)
2627 return ERR_PTR(-ENOMEM);
2628
2629 } else {
2630 b = root->commit_root;
2631 extent_buffer_get(b);
2632 }
2633 level = btrfs_header_level(b);
Liu Bof9ddfd02018-05-29 21:27:06 +08002634 /*
2635 * Ensure that all callers have set skip_locking when
2636 * p->search_commit_root = 1.
2637 */
2638 ASSERT(p->skip_locking == 1);
Liu Bo1fc28d82018-05-18 11:00:21 +08002639
2640 goto out;
2641 }
2642
2643 if (p->skip_locking) {
2644 b = btrfs_root_node(root);
2645 level = btrfs_header_level(b);
2646 goto out;
2647 }
2648
2649 /*
Liu Bo662c6532018-05-18 11:00:23 +08002650 * If the level is set to maximum, we can skip trying to get the read
2651 * lock.
Liu Bo1fc28d82018-05-18 11:00:21 +08002652 */
Liu Bo662c6532018-05-18 11:00:23 +08002653 if (write_lock_level < BTRFS_MAX_LEVEL) {
2654 /*
2655 * We don't know the level of the root node until we actually
2656 * have it read locked
2657 */
2658 b = btrfs_read_lock_root_node(root);
2659 level = btrfs_header_level(b);
2660 if (level > write_lock_level)
2661 goto out;
Liu Bo1fc28d82018-05-18 11:00:21 +08002662
Liu Bo662c6532018-05-18 11:00:23 +08002663 /* Whoops, must trade for write lock */
2664 btrfs_tree_read_unlock(b);
2665 free_extent_buffer(b);
2666 }
2667
Liu Bo1fc28d82018-05-18 11:00:21 +08002668 b = btrfs_lock_root_node(root);
2669 root_lock = BTRFS_WRITE_LOCK;
2670
2671 /* The level might have changed, check again */
2672 level = btrfs_header_level(b);
2673
2674out:
2675 p->nodes[level] = b;
2676 if (!p->skip_locking)
2677 p->locks[level] = root_lock;
2678 /*
2679 * Callers are responsible for dropping b's references.
2680 */
2681 return b;
2682}
2683
2684
Chris Masonc8c42862009-04-03 10:14:18 -04002685/*
Nikolay Borisov4271ece2017-12-13 09:38:14 +02002686 * btrfs_search_slot - look for a key in a tree and perform necessary
2687 * modifications to preserve tree invariants.
Chris Mason74123bd2007-02-02 11:05:29 -05002688 *
Nikolay Borisov4271ece2017-12-13 09:38:14 +02002689 * @trans: Handle of transaction, used when modifying the tree
2690 * @p: Holds all btree nodes along the search path
2691 * @root: The root node of the tree
2692 * @key: The key we are looking for
2693 * @ins_len: Indicates purpose of search, for inserts it is 1, for
2694 * deletions it's -1. 0 for plain searches
2695 * @cow: boolean should CoW operations be performed. Must always be 1
2696 * when modifying the tree.
Chris Mason97571fd2007-02-24 13:39:08 -05002697 *
Nikolay Borisov4271ece2017-12-13 09:38:14 +02002698 * If @ins_len > 0, nodes and leaves will be split as we walk down the tree.
2699 * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible)
2700 *
2701 * If @key is found, 0 is returned and you can find the item in the leaf level
2702 * of the path (level 0)
2703 *
2704 * If @key isn't found, 1 is returned and the leaf level of the path (level 0)
2705 * points to the slot where it should be inserted
2706 *
2707 * If an error is encountered while searching the tree a negative error number
2708 * is returned
Chris Mason74123bd2007-02-02 11:05:29 -05002709 */
Omar Sandoval310712b2017-01-17 23:24:37 -08002710int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2711 const struct btrfs_key *key, struct btrfs_path *p,
2712 int ins_len, int cow)
Chris Masonbe0e5c02007-01-26 15:51:26 -05002713{
Chris Mason5f39d392007-10-15 16:14:19 -04002714 struct extent_buffer *b;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002715 int slot;
2716 int ret;
Yan Zheng33c66f42009-07-22 09:59:00 -04002717 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002718 int level;
Chris Mason925baed2008-06-25 16:01:30 -04002719 int lowest_unlock = 1;
Chris Masonbd681512011-07-16 15:23:14 -04002720 /* everything at write_lock_level or lower must be write locked */
2721 int write_lock_level = 0;
Chris Mason9f3a7422007-08-07 15:52:19 -04002722 u8 lowest_level = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002723 int min_write_lock_level;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002724 int prev_cmp;
Chris Mason9f3a7422007-08-07 15:52:19 -04002725
Chris Mason6702ed42007-08-07 16:15:09 -04002726 lowest_level = p->lowest_level;
Chris Mason323ac952008-10-01 19:05:46 -04002727 WARN_ON(lowest_level && ins_len > 0);
Chris Mason22b0ebd2007-03-30 08:47:31 -04002728 WARN_ON(p->nodes[0] != NULL);
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002729 BUG_ON(!cow && ins_len);
Josef Bacik25179202008-10-29 14:49:05 -04002730
Chris Masonbd681512011-07-16 15:23:14 -04002731 if (ins_len < 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002732 lowest_unlock = 2;
Chris Mason65b51a02008-08-01 15:11:20 -04002733
Chris Masonbd681512011-07-16 15:23:14 -04002734 /* when we are removing items, we might have to go up to level
2735 * two as we update tree pointers Make sure we keep write
2736 * for those levels as well
2737 */
2738 write_lock_level = 2;
2739 } else if (ins_len > 0) {
2740 /*
2741 * for inserting items, make sure we have a write lock on
2742 * level 1 so we can update keys
2743 */
2744 write_lock_level = 1;
2745 }
2746
2747 if (!cow)
2748 write_lock_level = -1;
2749
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002750 if (cow && (p->keep_locks || p->lowest_level))
Chris Masonbd681512011-07-16 15:23:14 -04002751 write_lock_level = BTRFS_MAX_LEVEL;
2752
Chris Masonf7c79f32012-03-19 15:54:38 -04002753 min_write_lock_level = write_lock_level;
2754
Chris Masonbb803952007-03-01 12:04:21 -05002755again:
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002756 prev_cmp = -1;
Liu Bo1fc28d82018-05-18 11:00:21 +08002757 b = btrfs_search_slot_get_root(root, p, write_lock_level);
Filipe Mananabe6821f2018-12-11 10:19:45 +00002758 if (IS_ERR(b)) {
2759 ret = PTR_ERR(b);
2760 goto done;
2761 }
Chris Mason925baed2008-06-25 16:01:30 -04002762
Chris Masoneb60cea2007-02-02 09:18:22 -05002763 while (b) {
Chris Mason5f39d392007-10-15 16:14:19 -04002764 level = btrfs_header_level(b);
Chris Mason65b51a02008-08-01 15:11:20 -04002765
2766 /*
2767 * setup the path here so we can release it under lock
2768 * contention with the cow code
2769 */
Chris Mason02217ed2007-03-02 16:08:05 -05002770 if (cow) {
Nikolay Borisov9ea2c7c2017-12-12 11:14:49 +02002771 bool last_level = (level == (BTRFS_MAX_LEVEL - 1));
2772
Chris Masonc8c42862009-04-03 10:14:18 -04002773 /*
2774 * if we don't really need to cow this block
2775 * then we don't want to set the path blocking,
2776 * so we test it here
2777 */
Jeff Mahoney64c12922016-06-08 00:36:38 -04002778 if (!should_cow_block(trans, root, b)) {
2779 trans->dirty = true;
Chris Mason65b51a02008-08-01 15:11:20 -04002780 goto cow_done;
Jeff Mahoney64c12922016-06-08 00:36:38 -04002781 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002782
Chris Masonbd681512011-07-16 15:23:14 -04002783 /*
2784 * must have write locks on this node and the
2785 * parent
2786 */
Josef Bacik5124e002012-11-07 13:44:13 -05002787 if (level > write_lock_level ||
2788 (level + 1 > write_lock_level &&
2789 level + 1 < BTRFS_MAX_LEVEL &&
2790 p->nodes[level + 1])) {
Chris Masonbd681512011-07-16 15:23:14 -04002791 write_lock_level = level + 1;
2792 btrfs_release_path(p);
2793 goto again;
2794 }
2795
Filipe Manana160f4082014-07-28 19:37:17 +01002796 btrfs_set_path_blocking(p);
Nikolay Borisov9ea2c7c2017-12-12 11:14:49 +02002797 if (last_level)
2798 err = btrfs_cow_block(trans, root, b, NULL, 0,
2799 &b);
2800 else
2801 err = btrfs_cow_block(trans, root, b,
2802 p->nodes[level + 1],
2803 p->slots[level + 1], &b);
Yan Zheng33c66f42009-07-22 09:59:00 -04002804 if (err) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002805 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002806 goto done;
Chris Mason54aa1f42007-06-22 14:16:25 -04002807 }
Chris Mason02217ed2007-03-02 16:08:05 -05002808 }
Chris Mason65b51a02008-08-01 15:11:20 -04002809cow_done:
Chris Masoneb60cea2007-02-02 09:18:22 -05002810 p->nodes[level] = b;
Liu Bo52398342018-08-22 05:54:37 +08002811 /*
2812 * Leave path with blocking locks to avoid massive
2813 * lock context switch, this is made on purpose.
2814 */
Chris Masonb4ce94d2009-02-04 09:25:08 -05002815
2816 /*
2817 * we have a lock on b and as long as we aren't changing
2818 * the tree, there is no way to for the items in b to change.
2819 * It is safe to drop the lock on our parent before we
2820 * go through the expensive btree search on b.
2821 *
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002822 * If we're inserting or deleting (ins_len != 0), then we might
2823 * be changing slot zero, which may require changing the parent.
2824 * So, we can't drop the lock until after we know which slot
2825 * we're operating on.
Chris Masonb4ce94d2009-02-04 09:25:08 -05002826 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002827 if (!ins_len && !p->keep_locks) {
2828 int u = level + 1;
2829
2830 if (u < BTRFS_MAX_LEVEL && p->locks[u]) {
2831 btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]);
2832 p->locks[u] = 0;
2833 }
2834 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05002835
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002836 ret = key_search(b, key, level, &prev_cmp, &slot);
Liu Bo415b35a2016-06-17 19:16:21 -07002837 if (ret < 0)
2838 goto done;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002839
Chris Mason5f39d392007-10-15 16:14:19 -04002840 if (level != 0) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002841 int dec = 0;
2842 if (ret && slot > 0) {
2843 dec = 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002844 slot -= 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04002845 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002846 p->slots[level] = slot;
Yan Zheng33c66f42009-07-22 09:59:00 -04002847 err = setup_nodes_for_search(trans, root, p, b, level,
Chris Masonbd681512011-07-16 15:23:14 -04002848 ins_len, &write_lock_level);
Yan Zheng33c66f42009-07-22 09:59:00 -04002849 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002850 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002851 if (err) {
2852 ret = err;
Chris Masonc8c42862009-04-03 10:14:18 -04002853 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002854 }
Chris Masonc8c42862009-04-03 10:14:18 -04002855 b = p->nodes[level];
2856 slot = p->slots[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002857
Chris Masonbd681512011-07-16 15:23:14 -04002858 /*
2859 * slot 0 is special, if we change the key
2860 * we have to update the parent pointer
2861 * which means we must have a write lock
2862 * on the parent
2863 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002864 if (slot == 0 && ins_len &&
Chris Masonbd681512011-07-16 15:23:14 -04002865 write_lock_level < level + 1) {
2866 write_lock_level = level + 1;
2867 btrfs_release_path(p);
2868 goto again;
2869 }
2870
Chris Masonf7c79f32012-03-19 15:54:38 -04002871 unlock_up(p, level, lowest_unlock,
2872 min_write_lock_level, &write_lock_level);
Chris Masonf9efa9c2008-06-25 16:14:04 -04002873
Chris Mason925baed2008-06-25 16:01:30 -04002874 if (level == lowest_level) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002875 if (dec)
2876 p->slots[level]++;
Zheng Yan5b21f2e2008-09-26 10:05:38 -04002877 goto done;
Chris Mason925baed2008-06-25 16:01:30 -04002878 }
Chris Masonca7a79a2008-05-12 12:59:19 -04002879
Liu Bod07b8522017-01-30 12:23:42 -08002880 err = read_block_for_search(root, p, &b, level,
David Sterbacda79c52017-02-10 18:44:32 +01002881 slot, key);
Yan Zheng33c66f42009-07-22 09:59:00 -04002882 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002883 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002884 if (err) {
2885 ret = err;
Chris Mason76a05b32009-05-14 13:24:30 -04002886 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002887 }
Chris Mason76a05b32009-05-14 13:24:30 -04002888
Chris Masonb4ce94d2009-02-04 09:25:08 -05002889 if (!p->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04002890 level = btrfs_header_level(b);
2891 if (level <= write_lock_level) {
2892 err = btrfs_try_tree_write_lock(b);
2893 if (!err) {
2894 btrfs_set_path_blocking(p);
2895 btrfs_tree_lock(b);
Chris Masonbd681512011-07-16 15:23:14 -04002896 }
2897 p->locks[level] = BTRFS_WRITE_LOCK;
2898 } else {
Chris Masonf82c4582014-11-19 10:25:09 -08002899 err = btrfs_tree_read_lock_atomic(b);
Chris Masonbd681512011-07-16 15:23:14 -04002900 if (!err) {
2901 btrfs_set_path_blocking(p);
2902 btrfs_tree_read_lock(b);
Chris Masonbd681512011-07-16 15:23:14 -04002903 }
2904 p->locks[level] = BTRFS_READ_LOCK;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002905 }
Chris Masonbd681512011-07-16 15:23:14 -04002906 p->nodes[level] = b;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002907 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002908 } else {
2909 p->slots[level] = slot;
Yan Zheng87b29b22008-12-17 10:21:48 -05002910 if (ins_len > 0 &&
David Sterbae902baa2019-03-20 14:36:46 +01002911 btrfs_leaf_free_space(b) < ins_len) {
Chris Masonbd681512011-07-16 15:23:14 -04002912 if (write_lock_level < 1) {
2913 write_lock_level = 1;
2914 btrfs_release_path(p);
2915 goto again;
2916 }
2917
Chris Masonb4ce94d2009-02-04 09:25:08 -05002918 btrfs_set_path_blocking(p);
Yan Zheng33c66f42009-07-22 09:59:00 -04002919 err = split_leaf(trans, root, key,
2920 p, ins_len, ret == 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002921
Yan Zheng33c66f42009-07-22 09:59:00 -04002922 BUG_ON(err > 0);
2923 if (err) {
2924 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002925 goto done;
2926 }
Chris Mason5c680ed2007-02-22 11:39:13 -05002927 }
Chris Mason459931e2008-12-10 09:10:46 -05002928 if (!p->search_for_split)
Chris Masonf7c79f32012-03-19 15:54:38 -04002929 unlock_up(p, level, lowest_unlock,
Liu Bo4b6f8e92018-08-14 10:46:53 +08002930 min_write_lock_level, NULL);
Chris Mason65b51a02008-08-01 15:11:20 -04002931 goto done;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002932 }
2933 }
Chris Mason65b51a02008-08-01 15:11:20 -04002934 ret = 1;
2935done:
Chris Masonb4ce94d2009-02-04 09:25:08 -05002936 /*
2937 * we don't really know what they plan on doing with the path
2938 * from here on, so for now just mark it as blocking
2939 */
Chris Masonb9473432009-03-13 11:00:37 -04002940 if (!p->leave_spinning)
2941 btrfs_set_path_blocking(p);
Filipe Manana5f5bc6b2014-11-09 08:38:39 +00002942 if (ret < 0 && !p->skip_release_on_error)
David Sterbab3b4aa72011-04-21 01:20:15 +02002943 btrfs_release_path(p);
Chris Mason65b51a02008-08-01 15:11:20 -04002944 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002945}
2946
Chris Mason74123bd2007-02-02 11:05:29 -05002947/*
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002948 * Like btrfs_search_slot, this looks for a key in the given tree. It uses the
2949 * current state of the tree together with the operations recorded in the tree
2950 * modification log to search for the key in a previous version of this tree, as
2951 * denoted by the time_seq parameter.
2952 *
2953 * Naturally, there is no support for insert, delete or cow operations.
2954 *
2955 * The resulting path and return value will be set up as if we called
2956 * btrfs_search_slot at that point in time with ins_len and cow both set to 0.
2957 */
Omar Sandoval310712b2017-01-17 23:24:37 -08002958int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002959 struct btrfs_path *p, u64 time_seq)
2960{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002961 struct btrfs_fs_info *fs_info = root->fs_info;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002962 struct extent_buffer *b;
2963 int slot;
2964 int ret;
2965 int err;
2966 int level;
2967 int lowest_unlock = 1;
2968 u8 lowest_level = 0;
Josef Bacikd4b40872013-09-24 14:09:34 -04002969 int prev_cmp = -1;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002970
2971 lowest_level = p->lowest_level;
2972 WARN_ON(p->nodes[0] != NULL);
2973
2974 if (p->search_commit_root) {
2975 BUG_ON(time_seq);
2976 return btrfs_search_slot(NULL, root, key, p, 0, 0);
2977 }
2978
2979again:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002980 b = get_old_root(root, time_seq);
Nikolay Borisov315bed42018-09-13 11:35:10 +03002981 if (!b) {
2982 ret = -EIO;
2983 goto done;
2984 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002985 level = btrfs_header_level(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002986 p->locks[level] = BTRFS_READ_LOCK;
2987
2988 while (b) {
2989 level = btrfs_header_level(b);
2990 p->nodes[level] = b;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002991
2992 /*
2993 * we have a lock on b and as long as we aren't changing
2994 * the tree, there is no way to for the items in b to change.
2995 * It is safe to drop the lock on our parent before we
2996 * go through the expensive btree search on b.
2997 */
2998 btrfs_unlock_up_safe(p, level + 1);
2999
Josef Bacikd4b40872013-09-24 14:09:34 -04003000 /*
Nicholas D Steeves01327612016-05-19 21:18:45 -04003001 * Since we can unwind ebs we want to do a real search every
Josef Bacikd4b40872013-09-24 14:09:34 -04003002 * time.
3003 */
3004 prev_cmp = -1;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01003005 ret = key_search(b, key, level, &prev_cmp, &slot);
Filipe Mananacbca7d52019-02-18 16:57:26 +00003006 if (ret < 0)
3007 goto done;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003008
3009 if (level != 0) {
3010 int dec = 0;
3011 if (ret && slot > 0) {
3012 dec = 1;
3013 slot -= 1;
3014 }
3015 p->slots[level] = slot;
3016 unlock_up(p, level, lowest_unlock, 0, NULL);
3017
3018 if (level == lowest_level) {
3019 if (dec)
3020 p->slots[level]++;
3021 goto done;
3022 }
3023
Liu Bod07b8522017-01-30 12:23:42 -08003024 err = read_block_for_search(root, p, &b, level,
David Sterbacda79c52017-02-10 18:44:32 +01003025 slot, key);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003026 if (err == -EAGAIN)
3027 goto again;
3028 if (err) {
3029 ret = err;
3030 goto done;
3031 }
3032
3033 level = btrfs_header_level(b);
Chris Masonf82c4582014-11-19 10:25:09 -08003034 err = btrfs_tree_read_lock_atomic(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003035 if (!err) {
3036 btrfs_set_path_blocking(p);
3037 btrfs_tree_read_lock(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003038 }
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003039 b = tree_mod_log_rewind(fs_info, p, b, time_seq);
Josef Bacikdb7f3432013-08-07 14:54:37 -04003040 if (!b) {
3041 ret = -ENOMEM;
3042 goto done;
3043 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003044 p->locks[level] = BTRFS_READ_LOCK;
3045 p->nodes[level] = b;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003046 } else {
3047 p->slots[level] = slot;
3048 unlock_up(p, level, lowest_unlock, 0, NULL);
3049 goto done;
3050 }
3051 }
3052 ret = 1;
3053done:
3054 if (!p->leave_spinning)
3055 btrfs_set_path_blocking(p);
3056 if (ret < 0)
3057 btrfs_release_path(p);
3058
3059 return ret;
3060}
3061
3062/*
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003063 * helper to use instead of search slot if no exact match is needed but
3064 * instead the next or previous item should be returned.
3065 * When find_higher is true, the next higher item is returned, the next lower
3066 * otherwise.
3067 * When return_any and find_higher are both true, and no higher item is found,
3068 * return the next lower instead.
3069 * When return_any is true and find_higher is false, and no lower item is found,
3070 * return the next higher instead.
3071 * It returns 0 if any item is found, 1 if none is found (tree empty), and
3072 * < 0 on error
3073 */
3074int btrfs_search_slot_for_read(struct btrfs_root *root,
Omar Sandoval310712b2017-01-17 23:24:37 -08003075 const struct btrfs_key *key,
3076 struct btrfs_path *p, int find_higher,
3077 int return_any)
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003078{
3079 int ret;
3080 struct extent_buffer *leaf;
3081
3082again:
3083 ret = btrfs_search_slot(NULL, root, key, p, 0, 0);
3084 if (ret <= 0)
3085 return ret;
3086 /*
3087 * a return value of 1 means the path is at the position where the
3088 * item should be inserted. Normally this is the next bigger item,
3089 * but in case the previous item is the last in a leaf, path points
3090 * to the first free slot in the previous leaf, i.e. at an invalid
3091 * item.
3092 */
3093 leaf = p->nodes[0];
3094
3095 if (find_higher) {
3096 if (p->slots[0] >= btrfs_header_nritems(leaf)) {
3097 ret = btrfs_next_leaf(root, p);
3098 if (ret <= 0)
3099 return ret;
3100 if (!return_any)
3101 return 1;
3102 /*
3103 * no higher item found, return the next
3104 * lower instead
3105 */
3106 return_any = 0;
3107 find_higher = 0;
3108 btrfs_release_path(p);
3109 goto again;
3110 }
3111 } else {
Arne Jansene6793762011-09-13 11:18:10 +02003112 if (p->slots[0] == 0) {
3113 ret = btrfs_prev_leaf(root, p);
3114 if (ret < 0)
3115 return ret;
3116 if (!ret) {
Filipe David Borba Manana23c6bf62014-01-11 21:28:54 +00003117 leaf = p->nodes[0];
3118 if (p->slots[0] == btrfs_header_nritems(leaf))
3119 p->slots[0]--;
Arne Jansene6793762011-09-13 11:18:10 +02003120 return 0;
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003121 }
Arne Jansene6793762011-09-13 11:18:10 +02003122 if (!return_any)
3123 return 1;
3124 /*
3125 * no lower item found, return the next
3126 * higher instead
3127 */
3128 return_any = 0;
3129 find_higher = 1;
3130 btrfs_release_path(p);
3131 goto again;
3132 } else {
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003133 --p->slots[0];
3134 }
3135 }
3136 return 0;
3137}
3138
3139/*
Chris Mason74123bd2007-02-02 11:05:29 -05003140 * adjust the pointers going up the tree, starting at level
3141 * making sure the right key of each node is points to 'key'.
3142 * This is used after shifting pointers to the left, so it stops
3143 * fixing up pointers when a given leaf/node is not in slot 0 of the
3144 * higher levels
Chris Masonaa5d6be2007-02-28 16:35:06 -05003145 *
Chris Mason74123bd2007-02-02 11:05:29 -05003146 */
Nikolay Borisovb167fa92018-06-20 15:48:47 +03003147static void fixup_low_keys(struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003148 struct btrfs_disk_key *key, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003149{
3150 int i;
Chris Mason5f39d392007-10-15 16:14:19 -04003151 struct extent_buffer *t;
David Sterba0e82bcf2018-03-05 16:16:54 +01003152 int ret;
Chris Mason5f39d392007-10-15 16:14:19 -04003153
Chris Mason234b63a2007-03-13 10:46:10 -04003154 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05003155 int tslot = path->slots[i];
David Sterba0e82bcf2018-03-05 16:16:54 +01003156
Chris Masoneb60cea2007-02-02 09:18:22 -05003157 if (!path->nodes[i])
Chris Masonbe0e5c02007-01-26 15:51:26 -05003158 break;
Chris Mason5f39d392007-10-15 16:14:19 -04003159 t = path->nodes[i];
David Sterba0e82bcf2018-03-05 16:16:54 +01003160 ret = tree_mod_log_insert_key(t, tslot, MOD_LOG_KEY_REPLACE,
3161 GFP_ATOMIC);
3162 BUG_ON(ret < 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003163 btrfs_set_node_key(t, key, tslot);
Chris Masond6025572007-03-30 14:27:56 -04003164 btrfs_mark_buffer_dirty(path->nodes[i]);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003165 if (tslot != 0)
3166 break;
3167 }
3168}
3169
Chris Mason74123bd2007-02-02 11:05:29 -05003170/*
Zheng Yan31840ae2008-09-23 13:14:14 -04003171 * update item key.
3172 *
3173 * This function isn't completely safe. It's the caller's responsibility
3174 * that the new key won't break the order
3175 */
Daniel Dresslerb7a03652014-11-12 13:43:09 +09003176void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info,
3177 struct btrfs_path *path,
Omar Sandoval310712b2017-01-17 23:24:37 -08003178 const struct btrfs_key *new_key)
Zheng Yan31840ae2008-09-23 13:14:14 -04003179{
3180 struct btrfs_disk_key disk_key;
3181 struct extent_buffer *eb;
3182 int slot;
3183
3184 eb = path->nodes[0];
3185 slot = path->slots[0];
3186 if (slot > 0) {
3187 btrfs_item_key(eb, &disk_key, slot - 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003188 BUG_ON(comp_keys(&disk_key, new_key) >= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003189 }
3190 if (slot < btrfs_header_nritems(eb) - 1) {
3191 btrfs_item_key(eb, &disk_key, slot + 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003192 BUG_ON(comp_keys(&disk_key, new_key) <= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003193 }
3194
3195 btrfs_cpu_key_to_disk(&disk_key, new_key);
3196 btrfs_set_item_key(eb, &disk_key, slot);
3197 btrfs_mark_buffer_dirty(eb);
3198 if (slot == 0)
Nikolay Borisovb167fa92018-06-20 15:48:47 +03003199 fixup_low_keys(path, &disk_key, 1);
Zheng Yan31840ae2008-09-23 13:14:14 -04003200}
3201
3202/*
Chris Mason74123bd2007-02-02 11:05:29 -05003203 * try to push data from one node into the next node left in the
Chris Mason79f95c82007-03-01 15:16:26 -05003204 * tree.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003205 *
3206 * returns 0 if some ptrs were pushed left, < 0 if there was some horrible
3207 * error, and > 0 if there was no room in the left hand block.
Chris Mason74123bd2007-02-02 11:05:29 -05003208 */
Chris Mason98ed5172008-01-03 10:01:48 -05003209static int push_node_left(struct btrfs_trans_handle *trans,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04003210 struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -04003211 struct extent_buffer *src, int empty)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003212{
David Sterbad30a6682019-03-20 14:16:45 +01003213 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003214 int push_items = 0;
Chris Masonbb803952007-03-01 12:04:21 -05003215 int src_nritems;
3216 int dst_nritems;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003217 int ret = 0;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003218
Chris Mason5f39d392007-10-15 16:14:19 -04003219 src_nritems = btrfs_header_nritems(src);
3220 dst_nritems = btrfs_header_nritems(dst);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003221 push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems;
Chris Mason7bb86312007-12-11 09:25:06 -05003222 WARN_ON(btrfs_header_generation(src) != trans->transid);
3223 WARN_ON(btrfs_header_generation(dst) != trans->transid);
Chris Mason54aa1f42007-06-22 14:16:25 -04003224
Chris Masonbce4eae2008-04-24 14:42:46 -04003225 if (!empty && src_nritems <= 8)
Chris Mason971a1f62008-04-24 10:54:32 -04003226 return 1;
3227
Chris Masond3977122009-01-05 21:25:51 -05003228 if (push_items <= 0)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003229 return 1;
3230
Chris Masonbce4eae2008-04-24 14:42:46 -04003231 if (empty) {
Chris Mason971a1f62008-04-24 10:54:32 -04003232 push_items = min(src_nritems, push_items);
Chris Masonbce4eae2008-04-24 14:42:46 -04003233 if (push_items < src_nritems) {
3234 /* leave at least 8 pointers in the node if
3235 * we aren't going to empty it
3236 */
3237 if (src_nritems - push_items < 8) {
3238 if (push_items <= 8)
3239 return 1;
3240 push_items -= 8;
3241 }
3242 }
3243 } else
3244 push_items = min(src_nritems - 8, push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003245
David Sterbaed874f02019-03-20 14:22:04 +01003246 ret = tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003247 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04003248 btrfs_abort_transaction(trans, ret);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003249 return ret;
3250 }
Chris Mason5f39d392007-10-15 16:14:19 -04003251 copy_extent_buffer(dst, src,
3252 btrfs_node_key_ptr_offset(dst_nritems),
3253 btrfs_node_key_ptr_offset(0),
Chris Masond3977122009-01-05 21:25:51 -05003254 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason5f39d392007-10-15 16:14:19 -04003255
Chris Masonbb803952007-03-01 12:04:21 -05003256 if (push_items < src_nritems) {
Jan Schmidt57911b82012-10-19 09:22:03 +02003257 /*
David Sterbabf1d3422018-03-05 15:47:39 +01003258 * Don't call tree_mod_log_insert_move here, key removal was
3259 * already fully logged by tree_mod_log_eb_copy above.
Jan Schmidt57911b82012-10-19 09:22:03 +02003260 */
Chris Mason5f39d392007-10-15 16:14:19 -04003261 memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0),
3262 btrfs_node_key_ptr_offset(push_items),
3263 (src_nritems - push_items) *
3264 sizeof(struct btrfs_key_ptr));
Chris Masonbb803952007-03-01 12:04:21 -05003265 }
Chris Mason5f39d392007-10-15 16:14:19 -04003266 btrfs_set_header_nritems(src, src_nritems - push_items);
3267 btrfs_set_header_nritems(dst, dst_nritems + push_items);
3268 btrfs_mark_buffer_dirty(src);
3269 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003270
Chris Masonbb803952007-03-01 12:04:21 -05003271 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003272}
3273
Chris Mason97571fd2007-02-24 13:39:08 -05003274/*
Chris Mason79f95c82007-03-01 15:16:26 -05003275 * try to push data from one node into the next node right in the
3276 * tree.
3277 *
3278 * returns 0 if some ptrs were pushed, < 0 if there was some horrible
3279 * error, and > 0 if there was no room in the right hand block.
3280 *
3281 * this will only push up to 1/2 the contents of the left node over
3282 */
Chris Mason5f39d392007-10-15 16:14:19 -04003283static int balance_node_right(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04003284 struct extent_buffer *dst,
3285 struct extent_buffer *src)
Chris Mason79f95c82007-03-01 15:16:26 -05003286{
David Sterba55d32ed2019-03-20 14:18:06 +01003287 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Mason79f95c82007-03-01 15:16:26 -05003288 int push_items = 0;
3289 int max_push;
3290 int src_nritems;
3291 int dst_nritems;
3292 int ret = 0;
Chris Mason79f95c82007-03-01 15:16:26 -05003293
Chris Mason7bb86312007-12-11 09:25:06 -05003294 WARN_ON(btrfs_header_generation(src) != trans->transid);
3295 WARN_ON(btrfs_header_generation(dst) != trans->transid);
3296
Chris Mason5f39d392007-10-15 16:14:19 -04003297 src_nritems = btrfs_header_nritems(src);
3298 dst_nritems = btrfs_header_nritems(dst);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003299 push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems;
Chris Masond3977122009-01-05 21:25:51 -05003300 if (push_items <= 0)
Chris Mason79f95c82007-03-01 15:16:26 -05003301 return 1;
Chris Masonbce4eae2008-04-24 14:42:46 -04003302
Chris Masond3977122009-01-05 21:25:51 -05003303 if (src_nritems < 4)
Chris Masonbce4eae2008-04-24 14:42:46 -04003304 return 1;
Chris Mason79f95c82007-03-01 15:16:26 -05003305
3306 max_push = src_nritems / 2 + 1;
3307 /* don't try to empty the node */
Chris Masond3977122009-01-05 21:25:51 -05003308 if (max_push >= src_nritems)
Chris Mason79f95c82007-03-01 15:16:26 -05003309 return 1;
Yan252c38f2007-08-29 09:11:44 -04003310
Chris Mason79f95c82007-03-01 15:16:26 -05003311 if (max_push < push_items)
3312 push_items = max_push;
3313
David Sterbabf1d3422018-03-05 15:47:39 +01003314 ret = tree_mod_log_insert_move(dst, push_items, 0, dst_nritems);
3315 BUG_ON(ret < 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003316 memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items),
3317 btrfs_node_key_ptr_offset(0),
3318 (dst_nritems) *
3319 sizeof(struct btrfs_key_ptr));
Chris Masond6025572007-03-30 14:27:56 -04003320
David Sterbaed874f02019-03-20 14:22:04 +01003321 ret = tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items,
3322 push_items);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003323 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04003324 btrfs_abort_transaction(trans, ret);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003325 return ret;
3326 }
Chris Mason5f39d392007-10-15 16:14:19 -04003327 copy_extent_buffer(dst, src,
3328 btrfs_node_key_ptr_offset(0),
3329 btrfs_node_key_ptr_offset(src_nritems - push_items),
Chris Masond3977122009-01-05 21:25:51 -05003330 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason79f95c82007-03-01 15:16:26 -05003331
Chris Mason5f39d392007-10-15 16:14:19 -04003332 btrfs_set_header_nritems(src, src_nritems - push_items);
3333 btrfs_set_header_nritems(dst, dst_nritems + push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003334
Chris Mason5f39d392007-10-15 16:14:19 -04003335 btrfs_mark_buffer_dirty(src);
3336 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003337
Chris Mason79f95c82007-03-01 15:16:26 -05003338 return ret;
3339}
3340
3341/*
Chris Mason97571fd2007-02-24 13:39:08 -05003342 * helper function to insert a new root level in the tree.
3343 * A new node is allocated, and a single item is inserted to
3344 * point to the existing root
Chris Masonaa5d6be2007-02-28 16:35:06 -05003345 *
3346 * returns zero on success or < 0 on failure.
Chris Mason97571fd2007-02-24 13:39:08 -05003347 */
Chris Masond3977122009-01-05 21:25:51 -05003348static noinline int insert_new_root(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04003349 struct btrfs_root *root,
Liu Bofdd99c72013-05-22 12:06:51 +00003350 struct btrfs_path *path, int level)
Chris Mason5c680ed2007-02-22 11:39:13 -05003351{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003352 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason7bb86312007-12-11 09:25:06 -05003353 u64 lower_gen;
Chris Mason5f39d392007-10-15 16:14:19 -04003354 struct extent_buffer *lower;
3355 struct extent_buffer *c;
Chris Mason925baed2008-06-25 16:01:30 -04003356 struct extent_buffer *old;
Chris Mason5f39d392007-10-15 16:14:19 -04003357 struct btrfs_disk_key lower_key;
David Sterbad9d19a02018-03-05 16:35:29 +01003358 int ret;
Chris Mason5c680ed2007-02-22 11:39:13 -05003359
3360 BUG_ON(path->nodes[level]);
3361 BUG_ON(path->nodes[level-1] != root->node);
3362
Chris Mason7bb86312007-12-11 09:25:06 -05003363 lower = path->nodes[level-1];
3364 if (level == 1)
3365 btrfs_item_key(lower, &lower_key, 0);
3366 else
3367 btrfs_node_key(lower, &lower_key, 0);
3368
Filipe Mananaa6279472019-01-25 11:48:51 +00003369 c = alloc_tree_block_no_bg_flush(trans, root, 0, &lower_key, level,
3370 root->node->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003371 if (IS_ERR(c))
3372 return PTR_ERR(c);
Chris Mason925baed2008-06-25 16:01:30 -04003373
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003374 root_add_used(root, fs_info->nodesize);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003375
Chris Mason5f39d392007-10-15 16:14:19 -04003376 btrfs_set_header_nritems(c, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003377 btrfs_set_node_key(c, &lower_key, 0);
Chris Masondb945352007-10-15 16:15:53 -04003378 btrfs_set_node_blockptr(c, 0, lower->start);
Chris Mason7bb86312007-12-11 09:25:06 -05003379 lower_gen = btrfs_header_generation(lower);
Zheng Yan31840ae2008-09-23 13:14:14 -04003380 WARN_ON(lower_gen != trans->transid);
Chris Mason7bb86312007-12-11 09:25:06 -05003381
3382 btrfs_set_node_ptr_generation(c, 0, lower_gen);
Chris Mason5f39d392007-10-15 16:14:19 -04003383
3384 btrfs_mark_buffer_dirty(c);
Chris Masond5719762007-03-23 10:01:08 -04003385
Chris Mason925baed2008-06-25 16:01:30 -04003386 old = root->node;
David Sterbad9d19a02018-03-05 16:35:29 +01003387 ret = tree_mod_log_insert_root(root->node, c, 0);
3388 BUG_ON(ret < 0);
Chris Mason240f62c2011-03-23 14:54:42 -04003389 rcu_assign_pointer(root->node, c);
Chris Mason925baed2008-06-25 16:01:30 -04003390
3391 /* the super has an extra ref to root->node */
3392 free_extent_buffer(old);
3393
Chris Mason0b86a832008-03-24 15:01:56 -04003394 add_root_to_dirty_list(root);
Chris Mason5f39d392007-10-15 16:14:19 -04003395 extent_buffer_get(c);
3396 path->nodes[level] = c;
chandan95449a12015-01-15 12:22:03 +05303397 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Chris Mason5c680ed2007-02-22 11:39:13 -05003398 path->slots[level] = 0;
3399 return 0;
3400}
3401
Chris Mason74123bd2007-02-02 11:05:29 -05003402/*
3403 * worker function to insert a single pointer in a node.
3404 * the node should have enough room for the pointer already
Chris Mason97571fd2007-02-24 13:39:08 -05003405 *
Chris Mason74123bd2007-02-02 11:05:29 -05003406 * slot and level indicate where you want the key to go, and
3407 * blocknr is the block the key points to.
3408 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01003409static void insert_ptr(struct btrfs_trans_handle *trans,
David Sterba6ad3cf62019-03-20 14:32:45 +01003410 struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003411 struct btrfs_disk_key *key, u64 bytenr,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003412 int slot, int level)
Chris Mason74123bd2007-02-02 11:05:29 -05003413{
Chris Mason5f39d392007-10-15 16:14:19 -04003414 struct extent_buffer *lower;
Chris Mason74123bd2007-02-02 11:05:29 -05003415 int nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003416 int ret;
Chris Mason5c680ed2007-02-22 11:39:13 -05003417
3418 BUG_ON(!path->nodes[level]);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003419 btrfs_assert_tree_locked(path->nodes[level]);
Chris Mason5f39d392007-10-15 16:14:19 -04003420 lower = path->nodes[level];
3421 nritems = btrfs_header_nritems(lower);
Stoyan Gaydarovc2934982009-04-02 17:05:11 -04003422 BUG_ON(slot > nritems);
David Sterba6ad3cf62019-03-20 14:32:45 +01003423 BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info));
Chris Mason74123bd2007-02-02 11:05:29 -05003424 if (slot != nritems) {
David Sterbabf1d3422018-03-05 15:47:39 +01003425 if (level) {
3426 ret = tree_mod_log_insert_move(lower, slot + 1, slot,
David Sterbaa446a972018-03-05 15:26:29 +01003427 nritems - slot);
David Sterbabf1d3422018-03-05 15:47:39 +01003428 BUG_ON(ret < 0);
3429 }
Chris Mason5f39d392007-10-15 16:14:19 -04003430 memmove_extent_buffer(lower,
3431 btrfs_node_key_ptr_offset(slot + 1),
3432 btrfs_node_key_ptr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04003433 (nritems - slot) * sizeof(struct btrfs_key_ptr));
Chris Mason74123bd2007-02-02 11:05:29 -05003434 }
Jan Schmidtc3e06962012-06-21 11:01:06 +02003435 if (level) {
David Sterbae09c2ef2018-03-05 15:09:03 +01003436 ret = tree_mod_log_insert_key(lower, slot, MOD_LOG_KEY_ADD,
3437 GFP_NOFS);
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003438 BUG_ON(ret < 0);
3439 }
Chris Mason5f39d392007-10-15 16:14:19 -04003440 btrfs_set_node_key(lower, key, slot);
Chris Masondb945352007-10-15 16:15:53 -04003441 btrfs_set_node_blockptr(lower, slot, bytenr);
Chris Mason74493f72007-12-11 09:25:06 -05003442 WARN_ON(trans->transid == 0);
3443 btrfs_set_node_ptr_generation(lower, slot, trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003444 btrfs_set_header_nritems(lower, nritems + 1);
3445 btrfs_mark_buffer_dirty(lower);
Chris Mason74123bd2007-02-02 11:05:29 -05003446}
3447
Chris Mason97571fd2007-02-24 13:39:08 -05003448/*
3449 * split the node at the specified level in path in two.
3450 * The path is corrected to point to the appropriate node after the split
3451 *
3452 * Before splitting this tries to make some room in the node by pushing
3453 * left and right, if either one works, it returns right away.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003454 *
3455 * returns 0 on success and < 0 on failure
Chris Mason97571fd2007-02-24 13:39:08 -05003456 */
Chris Masone02119d2008-09-05 16:13:11 -04003457static noinline int split_node(struct btrfs_trans_handle *trans,
3458 struct btrfs_root *root,
3459 struct btrfs_path *path, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003460{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003461 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04003462 struct extent_buffer *c;
3463 struct extent_buffer *split;
3464 struct btrfs_disk_key disk_key;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003465 int mid;
Chris Mason5c680ed2007-02-22 11:39:13 -05003466 int ret;
Chris Mason7518a232007-03-12 12:01:18 -04003467 u32 c_nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003468
Chris Mason5f39d392007-10-15 16:14:19 -04003469 c = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05003470 WARN_ON(btrfs_header_generation(c) != trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003471 if (c == root->node) {
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003472 /*
Jan Schmidt90f8d622013-04-13 13:19:53 +00003473 * trying to split the root, lets make a new one
3474 *
Liu Bofdd99c72013-05-22 12:06:51 +00003475 * tree mod log: We don't log_removal old root in
Jan Schmidt90f8d622013-04-13 13:19:53 +00003476 * insert_new_root, because that root buffer will be kept as a
3477 * normal node. We are going to log removal of half of the
3478 * elements below with tree_mod_log_eb_copy. We're holding a
3479 * tree lock on the buffer, which is why we cannot race with
3480 * other tree_mod_log users.
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003481 */
Liu Bofdd99c72013-05-22 12:06:51 +00003482 ret = insert_new_root(trans, root, path, level + 1);
Chris Mason5c680ed2007-02-22 11:39:13 -05003483 if (ret)
3484 return ret;
Chris Masonb3612422009-05-13 19:12:15 -04003485 } else {
Chris Masone66f7092007-04-20 13:16:02 -04003486 ret = push_nodes_for_insert(trans, root, path, level);
Chris Mason5f39d392007-10-15 16:14:19 -04003487 c = path->nodes[level];
3488 if (!ret && btrfs_header_nritems(c) <
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003489 BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3)
Chris Masone66f7092007-04-20 13:16:02 -04003490 return 0;
Chris Mason54aa1f42007-06-22 14:16:25 -04003491 if (ret < 0)
3492 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003493 }
Chris Masone66f7092007-04-20 13:16:02 -04003494
Chris Mason5f39d392007-10-15 16:14:19 -04003495 c_nritems = btrfs_header_nritems(c);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003496 mid = (c_nritems + 1) / 2;
3497 btrfs_node_key(c, &disk_key, mid);
Chris Mason7bb86312007-12-11 09:25:06 -05003498
Filipe Mananaa6279472019-01-25 11:48:51 +00003499 split = alloc_tree_block_no_bg_flush(trans, root, 0, &disk_key, level,
3500 c->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003501 if (IS_ERR(split))
3502 return PTR_ERR(split);
Chris Mason54aa1f42007-06-22 14:16:25 -04003503
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003504 root_add_used(root, fs_info->nodesize);
Nikolay Borisovbc877d22018-06-18 14:13:19 +03003505 ASSERT(btrfs_header_level(c) == level);
Chris Mason5f39d392007-10-15 16:14:19 -04003506
David Sterbaed874f02019-03-20 14:22:04 +01003507 ret = tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003508 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04003509 btrfs_abort_transaction(trans, ret);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003510 return ret;
3511 }
Chris Mason5f39d392007-10-15 16:14:19 -04003512 copy_extent_buffer(split, c,
3513 btrfs_node_key_ptr_offset(0),
3514 btrfs_node_key_ptr_offset(mid),
3515 (c_nritems - mid) * sizeof(struct btrfs_key_ptr));
3516 btrfs_set_header_nritems(split, c_nritems - mid);
3517 btrfs_set_header_nritems(c, mid);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003518 ret = 0;
3519
Chris Mason5f39d392007-10-15 16:14:19 -04003520 btrfs_mark_buffer_dirty(c);
3521 btrfs_mark_buffer_dirty(split);
3522
David Sterba6ad3cf62019-03-20 14:32:45 +01003523 insert_ptr(trans, path, &disk_key, split->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003524 path->slots[level + 1] + 1, level + 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003525
Chris Mason5de08d72007-02-24 06:24:44 -05003526 if (path->slots[level] >= mid) {
Chris Mason5c680ed2007-02-22 11:39:13 -05003527 path->slots[level] -= mid;
Chris Mason925baed2008-06-25 16:01:30 -04003528 btrfs_tree_unlock(c);
Chris Mason5f39d392007-10-15 16:14:19 -04003529 free_extent_buffer(c);
3530 path->nodes[level] = split;
Chris Mason5c680ed2007-02-22 11:39:13 -05003531 path->slots[level + 1] += 1;
3532 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003533 btrfs_tree_unlock(split);
Chris Mason5f39d392007-10-15 16:14:19 -04003534 free_extent_buffer(split);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003535 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05003536 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003537}
3538
Chris Mason74123bd2007-02-02 11:05:29 -05003539/*
3540 * how many bytes are required to store the items in a leaf. start
3541 * and nr indicate which items in the leaf to check. This totals up the
3542 * space used both by the item structs and the item data
3543 */
Chris Mason5f39d392007-10-15 16:14:19 -04003544static int leaf_space_used(struct extent_buffer *l, int start, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003545{
Josef Bacik41be1f32012-10-15 13:43:18 -04003546 struct btrfs_item *start_item;
3547 struct btrfs_item *end_item;
3548 struct btrfs_map_token token;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003549 int data_len;
Chris Mason5f39d392007-10-15 16:14:19 -04003550 int nritems = btrfs_header_nritems(l);
Chris Masond4dbff92007-04-04 14:08:15 -04003551 int end = min(nritems, start + nr) - 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003552
3553 if (!nr)
3554 return 0;
Josef Bacik41be1f32012-10-15 13:43:18 -04003555 btrfs_init_map_token(&token);
Ross Kirkdd3cc162013-09-16 15:58:09 +01003556 start_item = btrfs_item_nr(start);
3557 end_item = btrfs_item_nr(end);
Josef Bacik41be1f32012-10-15 13:43:18 -04003558 data_len = btrfs_token_item_offset(l, start_item, &token) +
3559 btrfs_token_item_size(l, start_item, &token);
3560 data_len = data_len - btrfs_token_item_offset(l, end_item, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04003561 data_len += sizeof(struct btrfs_item) * nr;
Chris Masond4dbff92007-04-04 14:08:15 -04003562 WARN_ON(data_len < 0);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003563 return data_len;
3564}
3565
Chris Mason74123bd2007-02-02 11:05:29 -05003566/*
Chris Masond4dbff92007-04-04 14:08:15 -04003567 * The space between the end of the leaf items and
3568 * the start of the leaf data. IOW, how much room
3569 * the leaf has left for both items and data
3570 */
David Sterbae902baa2019-03-20 14:36:46 +01003571noinline int btrfs_leaf_free_space(struct extent_buffer *leaf)
Chris Masond4dbff92007-04-04 14:08:15 -04003572{
David Sterbae902baa2019-03-20 14:36:46 +01003573 struct btrfs_fs_info *fs_info = leaf->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04003574 int nritems = btrfs_header_nritems(leaf);
3575 int ret;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003576
3577 ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003578 if (ret < 0) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003579 btrfs_crit(fs_info,
3580 "leaf free space ret %d, leaf data size %lu, used %d nritems %d",
3581 ret,
3582 (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info),
3583 leaf_space_used(leaf, 0, nritems), nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003584 }
3585 return ret;
Chris Masond4dbff92007-04-04 14:08:15 -04003586}
3587
Chris Mason99d8f832010-07-07 10:51:48 -04003588/*
3589 * min slot controls the lowest index we're willing to push to the
3590 * right. We'll push up to and including min_slot, but no lower
3591 */
David Sterbaf72f0012019-03-20 14:39:45 +01003592static noinline int __push_leaf_right(struct btrfs_path *path,
Chris Mason44871b12009-03-13 10:04:31 -04003593 int data_size, int empty,
3594 struct extent_buffer *right,
Chris Mason99d8f832010-07-07 10:51:48 -04003595 int free_space, u32 left_nritems,
3596 u32 min_slot)
Chris Mason00ec4c52007-02-24 12:47:20 -05003597{
David Sterbaf72f0012019-03-20 14:39:45 +01003598 struct btrfs_fs_info *fs_info = right->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04003599 struct extent_buffer *left = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04003600 struct extent_buffer *upper = path->nodes[1];
Chris Masoncfed81a2012-03-03 07:40:03 -05003601 struct btrfs_map_token token;
Chris Mason5f39d392007-10-15 16:14:19 -04003602 struct btrfs_disk_key disk_key;
Chris Mason00ec4c52007-02-24 12:47:20 -05003603 int slot;
Chris Mason34a38212007-11-07 13:31:03 -05003604 u32 i;
Chris Mason00ec4c52007-02-24 12:47:20 -05003605 int push_space = 0;
3606 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003607 struct btrfs_item *item;
Chris Mason34a38212007-11-07 13:31:03 -05003608 u32 nr;
Chris Mason7518a232007-03-12 12:01:18 -04003609 u32 right_nritems;
Chris Mason5f39d392007-10-15 16:14:19 -04003610 u32 data_end;
Chris Masondb945352007-10-15 16:15:53 -04003611 u32 this_item_size;
Chris Mason00ec4c52007-02-24 12:47:20 -05003612
Chris Masoncfed81a2012-03-03 07:40:03 -05003613 btrfs_init_map_token(&token);
3614
Chris Mason34a38212007-11-07 13:31:03 -05003615 if (empty)
3616 nr = 0;
3617 else
Chris Mason99d8f832010-07-07 10:51:48 -04003618 nr = max_t(u32, 1, min_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003619
Zheng Yan31840ae2008-09-23 13:14:14 -04003620 if (path->slots[0] >= left_nritems)
Yan Zheng87b29b22008-12-17 10:21:48 -05003621 push_space += data_size;
Zheng Yan31840ae2008-09-23 13:14:14 -04003622
Chris Mason44871b12009-03-13 10:04:31 -04003623 slot = path->slots[1];
Chris Mason34a38212007-11-07 13:31:03 -05003624 i = left_nritems - 1;
3625 while (i >= nr) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003626 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003627
Zheng Yan31840ae2008-09-23 13:14:14 -04003628 if (!empty && push_items > 0) {
3629 if (path->slots[0] > i)
3630 break;
3631 if (path->slots[0] == i) {
David Sterbae902baa2019-03-20 14:36:46 +01003632 int space = btrfs_leaf_free_space(left);
3633
Zheng Yan31840ae2008-09-23 13:14:14 -04003634 if (space + push_space * 2 > free_space)
3635 break;
3636 }
3637 }
3638
Chris Mason00ec4c52007-02-24 12:47:20 -05003639 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003640 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003641
Chris Masondb945352007-10-15 16:15:53 -04003642 this_item_size = btrfs_item_size(left, item);
3643 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Mason00ec4c52007-02-24 12:47:20 -05003644 break;
Zheng Yan31840ae2008-09-23 13:14:14 -04003645
Chris Mason00ec4c52007-02-24 12:47:20 -05003646 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003647 push_space += this_item_size + sizeof(*item);
Chris Mason34a38212007-11-07 13:31:03 -05003648 if (i == 0)
3649 break;
3650 i--;
Chris Masondb945352007-10-15 16:15:53 -04003651 }
Chris Mason5f39d392007-10-15 16:14:19 -04003652
Chris Mason925baed2008-06-25 16:01:30 -04003653 if (push_items == 0)
3654 goto out_unlock;
Chris Mason5f39d392007-10-15 16:14:19 -04003655
Julia Lawall6c1500f2012-11-03 20:30:18 +00003656 WARN_ON(!empty && push_items == left_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003657
Chris Mason00ec4c52007-02-24 12:47:20 -05003658 /* push left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003659 right_nritems = btrfs_header_nritems(right);
Chris Mason34a38212007-11-07 13:31:03 -05003660
Chris Mason5f39d392007-10-15 16:14:19 -04003661 push_space = btrfs_item_end_nr(left, left_nritems - push_items);
David Sterba8f881e82019-03-20 11:33:10 +01003662 push_space -= leaf_data_end(left);
Chris Mason5f39d392007-10-15 16:14:19 -04003663
Chris Mason00ec4c52007-02-24 12:47:20 -05003664 /* make room in the right data area */
David Sterba8f881e82019-03-20 11:33:10 +01003665 data_end = leaf_data_end(right);
Chris Mason5f39d392007-10-15 16:14:19 -04003666 memmove_extent_buffer(right,
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003667 BTRFS_LEAF_DATA_OFFSET + data_end - push_space,
3668 BTRFS_LEAF_DATA_OFFSET + data_end,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003669 BTRFS_LEAF_DATA_SIZE(fs_info) - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04003670
Chris Mason00ec4c52007-02-24 12:47:20 -05003671 /* copy from the left data area */
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003672 copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET +
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003673 BTRFS_LEAF_DATA_SIZE(fs_info) - push_space,
David Sterba8f881e82019-03-20 11:33:10 +01003674 BTRFS_LEAF_DATA_OFFSET + leaf_data_end(left),
Chris Masond6025572007-03-30 14:27:56 -04003675 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003676
3677 memmove_extent_buffer(right, btrfs_item_nr_offset(push_items),
3678 btrfs_item_nr_offset(0),
3679 right_nritems * sizeof(struct btrfs_item));
3680
Chris Mason00ec4c52007-02-24 12:47:20 -05003681 /* copy the items from left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003682 copy_extent_buffer(right, left, btrfs_item_nr_offset(0),
3683 btrfs_item_nr_offset(left_nritems - push_items),
3684 push_items * sizeof(struct btrfs_item));
Chris Mason00ec4c52007-02-24 12:47:20 -05003685
3686 /* update the item pointers */
Chris Mason7518a232007-03-12 12:01:18 -04003687 right_nritems += push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003688 btrfs_set_header_nritems(right, right_nritems);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003689 push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
Chris Mason7518a232007-03-12 12:01:18 -04003690 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003691 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05003692 push_space -= btrfs_token_item_size(right, item, &token);
3693 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003694 }
3695
Chris Mason7518a232007-03-12 12:01:18 -04003696 left_nritems -= push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003697 btrfs_set_header_nritems(left, left_nritems);
Chris Mason00ec4c52007-02-24 12:47:20 -05003698
Chris Mason34a38212007-11-07 13:31:03 -05003699 if (left_nritems)
3700 btrfs_mark_buffer_dirty(left);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003701 else
David Sterba6a884d7d2019-03-20 14:30:02 +01003702 btrfs_clean_tree_block(left);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003703
Chris Mason5f39d392007-10-15 16:14:19 -04003704 btrfs_mark_buffer_dirty(right);
Chris Masona429e512007-04-18 16:15:28 -04003705
Chris Mason5f39d392007-10-15 16:14:19 -04003706 btrfs_item_key(right, &disk_key, 0);
3707 btrfs_set_node_key(upper, &disk_key, slot + 1);
Chris Masond6025572007-03-30 14:27:56 -04003708 btrfs_mark_buffer_dirty(upper);
Chris Mason02217ed2007-03-02 16:08:05 -05003709
Chris Mason00ec4c52007-02-24 12:47:20 -05003710 /* then fixup the leaf pointer in the path */
Chris Mason7518a232007-03-12 12:01:18 -04003711 if (path->slots[0] >= left_nritems) {
3712 path->slots[0] -= left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003713 if (btrfs_header_nritems(path->nodes[0]) == 0)
David Sterba6a884d7d2019-03-20 14:30:02 +01003714 btrfs_clean_tree_block(path->nodes[0]);
Chris Mason925baed2008-06-25 16:01:30 -04003715 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003716 free_extent_buffer(path->nodes[0]);
3717 path->nodes[0] = right;
Chris Mason00ec4c52007-02-24 12:47:20 -05003718 path->slots[1] += 1;
3719 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003720 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04003721 free_extent_buffer(right);
Chris Mason00ec4c52007-02-24 12:47:20 -05003722 }
3723 return 0;
Chris Mason925baed2008-06-25 16:01:30 -04003724
3725out_unlock:
3726 btrfs_tree_unlock(right);
3727 free_extent_buffer(right);
3728 return 1;
Chris Mason00ec4c52007-02-24 12:47:20 -05003729}
Chris Mason925baed2008-06-25 16:01:30 -04003730
Chris Mason00ec4c52007-02-24 12:47:20 -05003731/*
Chris Mason44871b12009-03-13 10:04:31 -04003732 * push some data in the path leaf to the right, trying to free up at
3733 * least data_size bytes. returns zero if the push worked, nonzero otherwise
3734 *
3735 * returns 1 if the push failed because the other node didn't have enough
3736 * room, 0 if everything worked out and < 0 if there were major errors.
Chris Mason99d8f832010-07-07 10:51:48 -04003737 *
3738 * this will push starting from min_slot to the end of the leaf. It won't
3739 * push any slot lower than min_slot
Chris Mason44871b12009-03-13 10:04:31 -04003740 */
3741static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003742 *root, struct btrfs_path *path,
3743 int min_data_size, int data_size,
3744 int empty, u32 min_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003745{
3746 struct extent_buffer *left = path->nodes[0];
3747 struct extent_buffer *right;
3748 struct extent_buffer *upper;
3749 int slot;
3750 int free_space;
3751 u32 left_nritems;
3752 int ret;
3753
3754 if (!path->nodes[1])
3755 return 1;
3756
3757 slot = path->slots[1];
3758 upper = path->nodes[1];
3759 if (slot >= btrfs_header_nritems(upper) - 1)
3760 return 1;
3761
3762 btrfs_assert_tree_locked(path->nodes[1]);
3763
David Sterbad0d20b02019-03-20 14:54:01 +01003764 right = read_node_slot(upper, slot + 1);
Liu Bofb770ae2016-07-05 12:10:14 -07003765 /*
3766 * slot + 1 is not valid or we fail to read the right node,
3767 * no big deal, just return.
3768 */
3769 if (IS_ERR(right))
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003770 return 1;
3771
Chris Mason44871b12009-03-13 10:04:31 -04003772 btrfs_tree_lock(right);
David Sterba8bead252018-04-04 02:03:48 +02003773 btrfs_set_lock_blocking_write(right);
Chris Mason44871b12009-03-13 10:04:31 -04003774
David Sterbae902baa2019-03-20 14:36:46 +01003775 free_space = btrfs_leaf_free_space(right);
Chris Mason44871b12009-03-13 10:04:31 -04003776 if (free_space < data_size)
3777 goto out_unlock;
3778
3779 /* cow and double check */
3780 ret = btrfs_cow_block(trans, root, right, upper,
3781 slot + 1, &right);
3782 if (ret)
3783 goto out_unlock;
3784
David Sterbae902baa2019-03-20 14:36:46 +01003785 free_space = btrfs_leaf_free_space(right);
Chris Mason44871b12009-03-13 10:04:31 -04003786 if (free_space < data_size)
3787 goto out_unlock;
3788
3789 left_nritems = btrfs_header_nritems(left);
3790 if (left_nritems == 0)
3791 goto out_unlock;
3792
Filipe David Borba Manana2ef1fed2013-12-04 22:17:39 +00003793 if (path->slots[0] == left_nritems && !empty) {
3794 /* Key greater than all keys in the leaf, right neighbor has
3795 * enough room for it and we're not emptying our leaf to delete
3796 * it, therefore use right neighbor to insert the new item and
Andrea Gelmini52042d82018-11-28 12:05:13 +01003797 * no need to touch/dirty our left leaf. */
Filipe David Borba Manana2ef1fed2013-12-04 22:17:39 +00003798 btrfs_tree_unlock(left);
3799 free_extent_buffer(left);
3800 path->nodes[0] = right;
3801 path->slots[0] = 0;
3802 path->slots[1]++;
3803 return 0;
3804 }
3805
David Sterbaf72f0012019-03-20 14:39:45 +01003806 return __push_leaf_right(path, min_data_size, empty,
Chris Mason99d8f832010-07-07 10:51:48 -04003807 right, free_space, left_nritems, min_slot);
Chris Mason44871b12009-03-13 10:04:31 -04003808out_unlock:
3809 btrfs_tree_unlock(right);
3810 free_extent_buffer(right);
3811 return 1;
3812}
3813
3814/*
Chris Mason74123bd2007-02-02 11:05:29 -05003815 * push some data in the path leaf to the left, trying to free up at
3816 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003817 *
3818 * max_slot can put a limit on how far into the leaf we'll push items. The
3819 * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the
3820 * items
Chris Mason74123bd2007-02-02 11:05:29 -05003821 */
David Sterba8087c192019-03-20 14:40:41 +01003822static noinline int __push_leaf_left(struct btrfs_path *path, int data_size,
Chris Mason44871b12009-03-13 10:04:31 -04003823 int empty, struct extent_buffer *left,
Chris Mason99d8f832010-07-07 10:51:48 -04003824 int free_space, u32 right_nritems,
3825 u32 max_slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003826{
David Sterba8087c192019-03-20 14:40:41 +01003827 struct btrfs_fs_info *fs_info = left->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04003828 struct btrfs_disk_key disk_key;
3829 struct extent_buffer *right = path->nodes[0];
Chris Masonbe0e5c02007-01-26 15:51:26 -05003830 int i;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003831 int push_space = 0;
3832 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003833 struct btrfs_item *item;
Chris Mason7518a232007-03-12 12:01:18 -04003834 u32 old_left_nritems;
Chris Mason34a38212007-11-07 13:31:03 -05003835 u32 nr;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003836 int ret = 0;
Chris Masondb945352007-10-15 16:15:53 -04003837 u32 this_item_size;
3838 u32 old_left_item_size;
Chris Masoncfed81a2012-03-03 07:40:03 -05003839 struct btrfs_map_token token;
3840
3841 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003842
Chris Mason34a38212007-11-07 13:31:03 -05003843 if (empty)
Chris Mason99d8f832010-07-07 10:51:48 -04003844 nr = min(right_nritems, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003845 else
Chris Mason99d8f832010-07-07 10:51:48 -04003846 nr = min(right_nritems - 1, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003847
3848 for (i = 0; i < nr; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003849 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003850
Zheng Yan31840ae2008-09-23 13:14:14 -04003851 if (!empty && push_items > 0) {
3852 if (path->slots[0] < i)
3853 break;
3854 if (path->slots[0] == i) {
David Sterbae902baa2019-03-20 14:36:46 +01003855 int space = btrfs_leaf_free_space(right);
3856
Zheng Yan31840ae2008-09-23 13:14:14 -04003857 if (space + push_space * 2 > free_space)
3858 break;
3859 }
3860 }
3861
Chris Masonbe0e5c02007-01-26 15:51:26 -05003862 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003863 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003864
3865 this_item_size = btrfs_item_size(right, item);
3866 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003867 break;
Chris Masondb945352007-10-15 16:15:53 -04003868
Chris Masonbe0e5c02007-01-26 15:51:26 -05003869 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003870 push_space += this_item_size + sizeof(*item);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003871 }
Chris Masondb945352007-10-15 16:15:53 -04003872
Chris Masonbe0e5c02007-01-26 15:51:26 -05003873 if (push_items == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04003874 ret = 1;
3875 goto out;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003876 }
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303877 WARN_ON(!empty && push_items == btrfs_header_nritems(right));
Chris Mason5f39d392007-10-15 16:14:19 -04003878
Chris Masonbe0e5c02007-01-26 15:51:26 -05003879 /* push data from right to left */
Chris Mason5f39d392007-10-15 16:14:19 -04003880 copy_extent_buffer(left, right,
3881 btrfs_item_nr_offset(btrfs_header_nritems(left)),
3882 btrfs_item_nr_offset(0),
3883 push_items * sizeof(struct btrfs_item));
3884
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003885 push_space = BTRFS_LEAF_DATA_SIZE(fs_info) -
Chris Masond3977122009-01-05 21:25:51 -05003886 btrfs_item_offset_nr(right, push_items - 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003887
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003888 copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET +
David Sterba8f881e82019-03-20 11:33:10 +01003889 leaf_data_end(left) - push_space,
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003890 BTRFS_LEAF_DATA_OFFSET +
Chris Mason5f39d392007-10-15 16:14:19 -04003891 btrfs_item_offset_nr(right, push_items - 1),
Chris Masond6025572007-03-30 14:27:56 -04003892 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003893 old_left_nritems = btrfs_header_nritems(left);
Yan Zheng87b29b22008-12-17 10:21:48 -05003894 BUG_ON(old_left_nritems <= 0);
Chris Masoneb60cea2007-02-02 09:18:22 -05003895
Chris Masondb945352007-10-15 16:15:53 -04003896 old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1);
Chris Mason0783fcf2007-03-12 20:12:07 -04003897 for (i = old_left_nritems; i < old_left_nritems + push_items; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04003898 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04003899
Ross Kirkdd3cc162013-09-16 15:58:09 +01003900 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003901
Chris Masoncfed81a2012-03-03 07:40:03 -05003902 ioff = btrfs_token_item_offset(left, item, &token);
3903 btrfs_set_token_item_offset(left, item,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003904 ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size),
Chris Masoncfed81a2012-03-03 07:40:03 -05003905 &token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003906 }
Chris Mason5f39d392007-10-15 16:14:19 -04003907 btrfs_set_header_nritems(left, old_left_nritems + push_items);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003908
3909 /* fixup right node */
Julia Lawall31b1a2b2012-11-03 10:58:34 +00003910 if (push_items > right_nritems)
3911 WARN(1, KERN_CRIT "push items %d nr %u\n", push_items,
Chris Masond3977122009-01-05 21:25:51 -05003912 right_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003913
Chris Mason34a38212007-11-07 13:31:03 -05003914 if (push_items < right_nritems) {
3915 push_space = btrfs_item_offset_nr(right, push_items - 1) -
David Sterba8f881e82019-03-20 11:33:10 +01003916 leaf_data_end(right);
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003917 memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET +
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003918 BTRFS_LEAF_DATA_SIZE(fs_info) - push_space,
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03003919 BTRFS_LEAF_DATA_OFFSET +
David Sterba8f881e82019-03-20 11:33:10 +01003920 leaf_data_end(right), push_space);
Chris Mason34a38212007-11-07 13:31:03 -05003921
3922 memmove_extent_buffer(right, btrfs_item_nr_offset(0),
Chris Mason5f39d392007-10-15 16:14:19 -04003923 btrfs_item_nr_offset(push_items),
3924 (btrfs_header_nritems(right) - push_items) *
3925 sizeof(struct btrfs_item));
Chris Mason34a38212007-11-07 13:31:03 -05003926 }
Yaneef1c492007-11-26 10:58:13 -05003927 right_nritems -= push_items;
3928 btrfs_set_header_nritems(right, right_nritems);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003929 push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
Chris Mason5f39d392007-10-15 16:14:19 -04003930 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003931 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003932
Chris Masoncfed81a2012-03-03 07:40:03 -05003933 push_space = push_space - btrfs_token_item_size(right,
3934 item, &token);
3935 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003936 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003937
Chris Mason5f39d392007-10-15 16:14:19 -04003938 btrfs_mark_buffer_dirty(left);
Chris Mason34a38212007-11-07 13:31:03 -05003939 if (right_nritems)
3940 btrfs_mark_buffer_dirty(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003941 else
David Sterba6a884d7d2019-03-20 14:30:02 +01003942 btrfs_clean_tree_block(right);
Chris Mason098f59c2007-05-11 11:33:21 -04003943
Chris Mason5f39d392007-10-15 16:14:19 -04003944 btrfs_item_key(right, &disk_key, 0);
Nikolay Borisovb167fa92018-06-20 15:48:47 +03003945 fixup_low_keys(path, &disk_key, 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003946
3947 /* then fixup the leaf pointer in the path */
3948 if (path->slots[0] < push_items) {
3949 path->slots[0] += old_left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003950 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003951 free_extent_buffer(path->nodes[0]);
3952 path->nodes[0] = left;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003953 path->slots[1] -= 1;
3954 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003955 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04003956 free_extent_buffer(left);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003957 path->slots[0] -= push_items;
3958 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003959 BUG_ON(path->slots[0] < 0);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003960 return ret;
Chris Mason925baed2008-06-25 16:01:30 -04003961out:
3962 btrfs_tree_unlock(left);
3963 free_extent_buffer(left);
3964 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003965}
3966
Chris Mason74123bd2007-02-02 11:05:29 -05003967/*
Chris Mason44871b12009-03-13 10:04:31 -04003968 * push some data in the path leaf to the left, trying to free up at
3969 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003970 *
3971 * max_slot can put a limit on how far into the leaf we'll push items. The
3972 * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the
3973 * items
Chris Mason44871b12009-03-13 10:04:31 -04003974 */
3975static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003976 *root, struct btrfs_path *path, int min_data_size,
3977 int data_size, int empty, u32 max_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003978{
3979 struct extent_buffer *right = path->nodes[0];
3980 struct extent_buffer *left;
3981 int slot;
3982 int free_space;
3983 u32 right_nritems;
3984 int ret = 0;
3985
3986 slot = path->slots[1];
3987 if (slot == 0)
3988 return 1;
3989 if (!path->nodes[1])
3990 return 1;
3991
3992 right_nritems = btrfs_header_nritems(right);
3993 if (right_nritems == 0)
3994 return 1;
3995
3996 btrfs_assert_tree_locked(path->nodes[1]);
3997
David Sterbad0d20b02019-03-20 14:54:01 +01003998 left = read_node_slot(path->nodes[1], slot - 1);
Liu Bofb770ae2016-07-05 12:10:14 -07003999 /*
4000 * slot - 1 is not valid or we fail to read the left node,
4001 * no big deal, just return.
4002 */
4003 if (IS_ERR(left))
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00004004 return 1;
4005
Chris Mason44871b12009-03-13 10:04:31 -04004006 btrfs_tree_lock(left);
David Sterba8bead252018-04-04 02:03:48 +02004007 btrfs_set_lock_blocking_write(left);
Chris Mason44871b12009-03-13 10:04:31 -04004008
David Sterbae902baa2019-03-20 14:36:46 +01004009 free_space = btrfs_leaf_free_space(left);
Chris Mason44871b12009-03-13 10:04:31 -04004010 if (free_space < data_size) {
4011 ret = 1;
4012 goto out;
4013 }
4014
4015 /* cow and double check */
4016 ret = btrfs_cow_block(trans, root, left,
4017 path->nodes[1], slot - 1, &left);
4018 if (ret) {
4019 /* we hit -ENOSPC, but it isn't fatal here */
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004020 if (ret == -ENOSPC)
4021 ret = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004022 goto out;
4023 }
4024
David Sterbae902baa2019-03-20 14:36:46 +01004025 free_space = btrfs_leaf_free_space(left);
Chris Mason44871b12009-03-13 10:04:31 -04004026 if (free_space < data_size) {
4027 ret = 1;
4028 goto out;
4029 }
4030
David Sterba8087c192019-03-20 14:40:41 +01004031 return __push_leaf_left(path, min_data_size,
Chris Mason99d8f832010-07-07 10:51:48 -04004032 empty, left, free_space, right_nritems,
4033 max_slot);
Chris Mason44871b12009-03-13 10:04:31 -04004034out:
4035 btrfs_tree_unlock(left);
4036 free_extent_buffer(left);
4037 return ret;
4038}
4039
4040/*
Chris Mason74123bd2007-02-02 11:05:29 -05004041 * split the path's leaf in two, making sure there is at least data_size
4042 * available for the resulting leaf level of the path.
4043 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004044static noinline void copy_for_split(struct btrfs_trans_handle *trans,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004045 struct btrfs_path *path,
4046 struct extent_buffer *l,
4047 struct extent_buffer *right,
4048 int slot, int mid, int nritems)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004049{
David Sterba94f94ad2019-03-20 14:42:33 +01004050 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004051 int data_copy_size;
4052 int rt_data_off;
4053 int i;
Chris Masond4dbff92007-04-04 14:08:15 -04004054 struct btrfs_disk_key disk_key;
Chris Masoncfed81a2012-03-03 07:40:03 -05004055 struct btrfs_map_token token;
4056
4057 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004058
Chris Mason5f39d392007-10-15 16:14:19 -04004059 nritems = nritems - mid;
4060 btrfs_set_header_nritems(right, nritems);
David Sterba8f881e82019-03-20 11:33:10 +01004061 data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(l);
Chris Mason5f39d392007-10-15 16:14:19 -04004062
4063 copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
4064 btrfs_item_nr_offset(mid),
4065 nritems * sizeof(struct btrfs_item));
4066
4067 copy_extent_buffer(right, l,
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03004068 BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) -
4069 data_copy_size, BTRFS_LEAF_DATA_OFFSET +
David Sterba8f881e82019-03-20 11:33:10 +01004070 leaf_data_end(l), data_copy_size);
Chris Mason74123bd2007-02-02 11:05:29 -05004071
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004072 rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_end_nr(l, mid);
Chris Mason5f39d392007-10-15 16:14:19 -04004073
4074 for (i = 0; i < nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01004075 struct btrfs_item *item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004076 u32 ioff;
4077
Chris Masoncfed81a2012-03-03 07:40:03 -05004078 ioff = btrfs_token_item_offset(right, item, &token);
4079 btrfs_set_token_item_offset(right, item,
4080 ioff + rt_data_off, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004081 }
Chris Mason74123bd2007-02-02 11:05:29 -05004082
Chris Mason5f39d392007-10-15 16:14:19 -04004083 btrfs_set_header_nritems(l, mid);
Chris Mason5f39d392007-10-15 16:14:19 -04004084 btrfs_item_key(right, &disk_key, 0);
David Sterba6ad3cf62019-03-20 14:32:45 +01004085 insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04004086
4087 btrfs_mark_buffer_dirty(right);
4088 btrfs_mark_buffer_dirty(l);
Chris Masoneb60cea2007-02-02 09:18:22 -05004089 BUG_ON(path->slots[0] != slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004090
Chris Masonbe0e5c02007-01-26 15:51:26 -05004091 if (mid <= slot) {
Chris Mason925baed2008-06-25 16:01:30 -04004092 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04004093 free_extent_buffer(path->nodes[0]);
4094 path->nodes[0] = right;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004095 path->slots[0] -= mid;
4096 path->slots[1] += 1;
Chris Mason925baed2008-06-25 16:01:30 -04004097 } else {
4098 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04004099 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04004100 }
Chris Mason5f39d392007-10-15 16:14:19 -04004101
Chris Masoneb60cea2007-02-02 09:18:22 -05004102 BUG_ON(path->slots[0] < 0);
Chris Mason44871b12009-03-13 10:04:31 -04004103}
4104
4105/*
Chris Mason99d8f832010-07-07 10:51:48 -04004106 * double splits happen when we need to insert a big item in the middle
4107 * of a leaf. A double split can leave us with 3 mostly empty leaves:
4108 * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ]
4109 * A B C
4110 *
4111 * We avoid this by trying to push the items on either side of our target
4112 * into the adjacent leaves. If all goes well we can avoid the double split
4113 * completely.
4114 */
4115static noinline int push_for_double_split(struct btrfs_trans_handle *trans,
4116 struct btrfs_root *root,
4117 struct btrfs_path *path,
4118 int data_size)
4119{
4120 int ret;
4121 int progress = 0;
4122 int slot;
4123 u32 nritems;
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004124 int space_needed = data_size;
Chris Mason99d8f832010-07-07 10:51:48 -04004125
4126 slot = path->slots[0];
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004127 if (slot < btrfs_header_nritems(path->nodes[0]))
David Sterbae902baa2019-03-20 14:36:46 +01004128 space_needed -= btrfs_leaf_free_space(path->nodes[0]);
Chris Mason99d8f832010-07-07 10:51:48 -04004129
4130 /*
4131 * try to push all the items after our slot into the
4132 * right leaf
4133 */
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004134 ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004135 if (ret < 0)
4136 return ret;
4137
4138 if (ret == 0)
4139 progress++;
4140
4141 nritems = btrfs_header_nritems(path->nodes[0]);
4142 /*
4143 * our goal is to get our slot at the start or end of a leaf. If
4144 * we've done so we're done
4145 */
4146 if (path->slots[0] == 0 || path->slots[0] == nritems)
4147 return 0;
4148
David Sterbae902baa2019-03-20 14:36:46 +01004149 if (btrfs_leaf_free_space(path->nodes[0]) >= data_size)
Chris Mason99d8f832010-07-07 10:51:48 -04004150 return 0;
4151
4152 /* try to push all the items before our slot into the next leaf */
4153 slot = path->slots[0];
Filipe Manana263d3992017-02-17 18:43:57 +00004154 space_needed = data_size;
4155 if (slot > 0)
David Sterbae902baa2019-03-20 14:36:46 +01004156 space_needed -= btrfs_leaf_free_space(path->nodes[0]);
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004157 ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004158 if (ret < 0)
4159 return ret;
4160
4161 if (ret == 0)
4162 progress++;
4163
4164 if (progress)
4165 return 0;
4166 return 1;
4167}
4168
4169/*
Chris Mason44871b12009-03-13 10:04:31 -04004170 * split the path's leaf in two, making sure there is at least data_size
4171 * available for the resulting leaf level of the path.
4172 *
4173 * returns 0 if all went well and < 0 on failure.
4174 */
4175static noinline int split_leaf(struct btrfs_trans_handle *trans,
4176 struct btrfs_root *root,
Omar Sandoval310712b2017-01-17 23:24:37 -08004177 const struct btrfs_key *ins_key,
Chris Mason44871b12009-03-13 10:04:31 -04004178 struct btrfs_path *path, int data_size,
4179 int extend)
4180{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004181 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004182 struct extent_buffer *l;
4183 u32 nritems;
4184 int mid;
4185 int slot;
4186 struct extent_buffer *right;
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004187 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason44871b12009-03-13 10:04:31 -04004188 int ret = 0;
4189 int wret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004190 int split;
Chris Mason44871b12009-03-13 10:04:31 -04004191 int num_doubles = 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004192 int tried_avoid_double = 0;
Chris Mason44871b12009-03-13 10:04:31 -04004193
Yan, Zhenga5719522009-09-24 09:17:31 -04004194 l = path->nodes[0];
4195 slot = path->slots[0];
4196 if (extend && data_size + btrfs_item_size_nr(l, slot) +
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004197 sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info))
Yan, Zhenga5719522009-09-24 09:17:31 -04004198 return -EOVERFLOW;
4199
Chris Mason44871b12009-03-13 10:04:31 -04004200 /* first try to make some room by pushing left and right */
Liu Bo33157e02013-05-22 12:07:06 +00004201 if (data_size && path->nodes[1]) {
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004202 int space_needed = data_size;
4203
4204 if (slot < btrfs_header_nritems(l))
David Sterbae902baa2019-03-20 14:36:46 +01004205 space_needed -= btrfs_leaf_free_space(l);
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004206
4207 wret = push_leaf_right(trans, root, path, space_needed,
4208 space_needed, 0, 0);
Chris Mason44871b12009-03-13 10:04:31 -04004209 if (wret < 0)
4210 return wret;
4211 if (wret) {
Filipe Manana263d3992017-02-17 18:43:57 +00004212 space_needed = data_size;
4213 if (slot > 0)
David Sterbae902baa2019-03-20 14:36:46 +01004214 space_needed -= btrfs_leaf_free_space(l);
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004215 wret = push_leaf_left(trans, root, path, space_needed,
4216 space_needed, 0, (u32)-1);
Chris Mason44871b12009-03-13 10:04:31 -04004217 if (wret < 0)
4218 return wret;
4219 }
4220 l = path->nodes[0];
4221
4222 /* did the pushes work? */
David Sterbae902baa2019-03-20 14:36:46 +01004223 if (btrfs_leaf_free_space(l) >= data_size)
Chris Mason44871b12009-03-13 10:04:31 -04004224 return 0;
4225 }
4226
4227 if (!path->nodes[1]) {
Liu Bofdd99c72013-05-22 12:06:51 +00004228 ret = insert_new_root(trans, root, path, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004229 if (ret)
4230 return ret;
4231 }
4232again:
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004233 split = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004234 l = path->nodes[0];
4235 slot = path->slots[0];
4236 nritems = btrfs_header_nritems(l);
4237 mid = (nritems + 1) / 2;
4238
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004239 if (mid <= slot) {
4240 if (nritems == 1 ||
4241 leaf_space_used(l, mid, nritems - mid) + data_size >
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004242 BTRFS_LEAF_DATA_SIZE(fs_info)) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004243 if (slot >= nritems) {
4244 split = 0;
4245 } else {
4246 mid = slot;
4247 if (mid != nritems &&
4248 leaf_space_used(l, mid, nritems - mid) +
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004249 data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004250 if (data_size && !tried_avoid_double)
4251 goto push_for_double;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004252 split = 2;
4253 }
4254 }
4255 }
4256 } else {
4257 if (leaf_space_used(l, 0, mid) + data_size >
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004258 BTRFS_LEAF_DATA_SIZE(fs_info)) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004259 if (!extend && data_size && slot == 0) {
4260 split = 0;
4261 } else if ((extend || !data_size) && slot == 0) {
4262 mid = 1;
4263 } else {
4264 mid = slot;
4265 if (mid != nritems &&
4266 leaf_space_used(l, mid, nritems - mid) +
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004267 data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004268 if (data_size && !tried_avoid_double)
4269 goto push_for_double;
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304270 split = 2;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004271 }
4272 }
4273 }
4274 }
4275
4276 if (split == 0)
4277 btrfs_cpu_key_to_disk(&disk_key, ins_key);
4278 else
4279 btrfs_item_key(l, &disk_key, mid);
4280
Filipe Mananaa6279472019-01-25 11:48:51 +00004281 right = alloc_tree_block_no_bg_flush(trans, root, 0, &disk_key, 0,
4282 l->start, 0);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004283 if (IS_ERR(right))
Chris Mason44871b12009-03-13 10:04:31 -04004284 return PTR_ERR(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004285
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004286 root_add_used(root, fs_info->nodesize);
Chris Mason44871b12009-03-13 10:04:31 -04004287
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004288 if (split == 0) {
4289 if (mid <= slot) {
4290 btrfs_set_header_nritems(right, 0);
David Sterba6ad3cf62019-03-20 14:32:45 +01004291 insert_ptr(trans, path, &disk_key,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004292 right->start, path->slots[1] + 1, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004293 btrfs_tree_unlock(path->nodes[0]);
4294 free_extent_buffer(path->nodes[0]);
4295 path->nodes[0] = right;
4296 path->slots[0] = 0;
4297 path->slots[1] += 1;
4298 } else {
4299 btrfs_set_header_nritems(right, 0);
David Sterba6ad3cf62019-03-20 14:32:45 +01004300 insert_ptr(trans, path, &disk_key,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004301 right->start, path->slots[1], 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004302 btrfs_tree_unlock(path->nodes[0]);
4303 free_extent_buffer(path->nodes[0]);
4304 path->nodes[0] = right;
4305 path->slots[0] = 0;
Jeff Mahoney143bede2012-03-01 14:56:26 +01004306 if (path->slots[1] == 0)
Nikolay Borisovb167fa92018-06-20 15:48:47 +03004307 fixup_low_keys(path, &disk_key, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004308 }
Liu Bo196e0242016-09-07 14:48:28 -07004309 /*
4310 * We create a new leaf 'right' for the required ins_len and
4311 * we'll do btrfs_mark_buffer_dirty() on this leaf after copying
4312 * the content of ins_len to 'right'.
4313 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004314 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004315 }
4316
David Sterba94f94ad2019-03-20 14:42:33 +01004317 copy_for_split(trans, path, l, right, slot, mid, nritems);
Chris Mason44871b12009-03-13 10:04:31 -04004318
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004319 if (split == 2) {
Chris Masoncc0c5532007-10-25 15:42:57 -04004320 BUG_ON(num_doubles != 0);
4321 num_doubles++;
4322 goto again;
Chris Mason3326d1b2007-10-15 16:18:25 -04004323 }
Chris Mason44871b12009-03-13 10:04:31 -04004324
Jeff Mahoney143bede2012-03-01 14:56:26 +01004325 return 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004326
4327push_for_double:
4328 push_for_double_split(trans, root, path, data_size);
4329 tried_avoid_double = 1;
David Sterbae902baa2019-03-20 14:36:46 +01004330 if (btrfs_leaf_free_space(path->nodes[0]) >= data_size)
Chris Mason99d8f832010-07-07 10:51:48 -04004331 return 0;
4332 goto again;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004333}
4334
Yan, Zhengad48fd752009-11-12 09:33:58 +00004335static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
4336 struct btrfs_root *root,
4337 struct btrfs_path *path, int ins_len)
Chris Mason459931e2008-12-10 09:10:46 -05004338{
Yan, Zhengad48fd752009-11-12 09:33:58 +00004339 struct btrfs_key key;
Chris Mason459931e2008-12-10 09:10:46 -05004340 struct extent_buffer *leaf;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004341 struct btrfs_file_extent_item *fi;
4342 u64 extent_len = 0;
4343 u32 item_size;
4344 int ret;
Chris Mason459931e2008-12-10 09:10:46 -05004345
4346 leaf = path->nodes[0];
Yan, Zhengad48fd752009-11-12 09:33:58 +00004347 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4348
4349 BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY &&
4350 key.type != BTRFS_EXTENT_CSUM_KEY);
4351
David Sterbae902baa2019-03-20 14:36:46 +01004352 if (btrfs_leaf_free_space(leaf) >= ins_len)
Yan, Zhengad48fd752009-11-12 09:33:58 +00004353 return 0;
Chris Mason459931e2008-12-10 09:10:46 -05004354
4355 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004356 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4357 fi = btrfs_item_ptr(leaf, path->slots[0],
4358 struct btrfs_file_extent_item);
4359 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
4360 }
David Sterbab3b4aa72011-04-21 01:20:15 +02004361 btrfs_release_path(path);
Chris Mason459931e2008-12-10 09:10:46 -05004362
Chris Mason459931e2008-12-10 09:10:46 -05004363 path->keep_locks = 1;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004364 path->search_for_split = 1;
4365 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
Chris Mason459931e2008-12-10 09:10:46 -05004366 path->search_for_split = 0;
Filipe Mananaa8df6fe2015-01-20 12:40:53 +00004367 if (ret > 0)
4368 ret = -EAGAIN;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004369 if (ret < 0)
4370 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004371
Yan, Zhengad48fd752009-11-12 09:33:58 +00004372 ret = -EAGAIN;
4373 leaf = path->nodes[0];
Filipe Mananaa8df6fe2015-01-20 12:40:53 +00004374 /* if our item isn't there, return now */
4375 if (item_size != btrfs_item_size_nr(leaf, path->slots[0]))
Yan, Zhengad48fd752009-11-12 09:33:58 +00004376 goto err;
4377
Chris Mason109f6ae2010-04-02 09:20:18 -04004378 /* the leaf has changed, it now has room. return now */
David Sterbae902baa2019-03-20 14:36:46 +01004379 if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len)
Chris Mason109f6ae2010-04-02 09:20:18 -04004380 goto err;
4381
Yan, Zhengad48fd752009-11-12 09:33:58 +00004382 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4383 fi = btrfs_item_ptr(leaf, path->slots[0],
4384 struct btrfs_file_extent_item);
4385 if (extent_len != btrfs_file_extent_num_bytes(leaf, fi))
4386 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004387 }
4388
Chris Masonb9473432009-03-13 11:00:37 -04004389 btrfs_set_path_blocking(path);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004390 ret = split_leaf(trans, root, &key, path, ins_len, 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004391 if (ret)
4392 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004393
Yan, Zhengad48fd752009-11-12 09:33:58 +00004394 path->keep_locks = 0;
Chris Masonb9473432009-03-13 11:00:37 -04004395 btrfs_unlock_up_safe(path, 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004396 return 0;
4397err:
4398 path->keep_locks = 0;
4399 return ret;
4400}
4401
David Sterba25263cd2019-03-20 14:44:57 +01004402static noinline int split_item(struct btrfs_path *path,
Omar Sandoval310712b2017-01-17 23:24:37 -08004403 const struct btrfs_key *new_key,
Yan, Zhengad48fd752009-11-12 09:33:58 +00004404 unsigned long split_offset)
4405{
4406 struct extent_buffer *leaf;
4407 struct btrfs_item *item;
4408 struct btrfs_item *new_item;
4409 int slot;
4410 char *buf;
4411 u32 nritems;
4412 u32 item_size;
4413 u32 orig_offset;
4414 struct btrfs_disk_key disk_key;
4415
Chris Masonb9473432009-03-13 11:00:37 -04004416 leaf = path->nodes[0];
David Sterbae902baa2019-03-20 14:36:46 +01004417 BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item));
Chris Masonb9473432009-03-13 11:00:37 -04004418
Chris Masonb4ce94d2009-02-04 09:25:08 -05004419 btrfs_set_path_blocking(path);
4420
Ross Kirkdd3cc162013-09-16 15:58:09 +01004421 item = btrfs_item_nr(path->slots[0]);
Chris Mason459931e2008-12-10 09:10:46 -05004422 orig_offset = btrfs_item_offset(leaf, item);
4423 item_size = btrfs_item_size(leaf, item);
4424
Chris Mason459931e2008-12-10 09:10:46 -05004425 buf = kmalloc(item_size, GFP_NOFS);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004426 if (!buf)
4427 return -ENOMEM;
4428
Chris Mason459931e2008-12-10 09:10:46 -05004429 read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf,
4430 path->slots[0]), item_size);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004431
Chris Mason459931e2008-12-10 09:10:46 -05004432 slot = path->slots[0] + 1;
Chris Mason459931e2008-12-10 09:10:46 -05004433 nritems = btrfs_header_nritems(leaf);
Chris Mason459931e2008-12-10 09:10:46 -05004434 if (slot != nritems) {
4435 /* shift the items */
4436 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1),
Yan, Zhengad48fd752009-11-12 09:33:58 +00004437 btrfs_item_nr_offset(slot),
4438 (nritems - slot) * sizeof(struct btrfs_item));
Chris Mason459931e2008-12-10 09:10:46 -05004439 }
4440
4441 btrfs_cpu_key_to_disk(&disk_key, new_key);
4442 btrfs_set_item_key(leaf, &disk_key, slot);
4443
Ross Kirkdd3cc162013-09-16 15:58:09 +01004444 new_item = btrfs_item_nr(slot);
Chris Mason459931e2008-12-10 09:10:46 -05004445
4446 btrfs_set_item_offset(leaf, new_item, orig_offset);
4447 btrfs_set_item_size(leaf, new_item, item_size - split_offset);
4448
4449 btrfs_set_item_offset(leaf, item,
4450 orig_offset + item_size - split_offset);
4451 btrfs_set_item_size(leaf, item, split_offset);
4452
4453 btrfs_set_header_nritems(leaf, nritems + 1);
4454
4455 /* write the data for the start of the original item */
4456 write_extent_buffer(leaf, buf,
4457 btrfs_item_ptr_offset(leaf, path->slots[0]),
4458 split_offset);
4459
4460 /* write the data for the new item */
4461 write_extent_buffer(leaf, buf + split_offset,
4462 btrfs_item_ptr_offset(leaf, slot),
4463 item_size - split_offset);
4464 btrfs_mark_buffer_dirty(leaf);
4465
David Sterbae902baa2019-03-20 14:36:46 +01004466 BUG_ON(btrfs_leaf_free_space(leaf) < 0);
Chris Mason459931e2008-12-10 09:10:46 -05004467 kfree(buf);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004468 return 0;
4469}
4470
4471/*
4472 * This function splits a single item into two items,
4473 * giving 'new_key' to the new item and splitting the
4474 * old one at split_offset (from the start of the item).
4475 *
4476 * The path may be released by this operation. After
4477 * the split, the path is pointing to the old item. The
4478 * new item is going to be in the same node as the old one.
4479 *
4480 * Note, the item being split must be smaller enough to live alone on
4481 * a tree block with room for one extra struct btrfs_item
4482 *
4483 * This allows us to split the item in place, keeping a lock on the
4484 * leaf the entire time.
4485 */
4486int btrfs_split_item(struct btrfs_trans_handle *trans,
4487 struct btrfs_root *root,
4488 struct btrfs_path *path,
Omar Sandoval310712b2017-01-17 23:24:37 -08004489 const struct btrfs_key *new_key,
Yan, Zhengad48fd752009-11-12 09:33:58 +00004490 unsigned long split_offset)
4491{
4492 int ret;
4493 ret = setup_leaf_for_split(trans, root, path,
4494 sizeof(struct btrfs_item));
4495 if (ret)
4496 return ret;
4497
David Sterba25263cd2019-03-20 14:44:57 +01004498 ret = split_item(path, new_key, split_offset);
Chris Mason459931e2008-12-10 09:10:46 -05004499 return ret;
4500}
4501
4502/*
Yan, Zhengad48fd752009-11-12 09:33:58 +00004503 * This function duplicate a item, giving 'new_key' to the new item.
4504 * It guarantees both items live in the same tree leaf and the new item
4505 * is contiguous with the original item.
4506 *
4507 * This allows us to split file extent in place, keeping a lock on the
4508 * leaf the entire time.
4509 */
4510int btrfs_duplicate_item(struct btrfs_trans_handle *trans,
4511 struct btrfs_root *root,
4512 struct btrfs_path *path,
Omar Sandoval310712b2017-01-17 23:24:37 -08004513 const struct btrfs_key *new_key)
Yan, Zhengad48fd752009-11-12 09:33:58 +00004514{
4515 struct extent_buffer *leaf;
4516 int ret;
4517 u32 item_size;
4518
4519 leaf = path->nodes[0];
4520 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
4521 ret = setup_leaf_for_split(trans, root, path,
4522 item_size + sizeof(struct btrfs_item));
4523 if (ret)
4524 return ret;
4525
4526 path->slots[0]++;
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004527 setup_items_for_insert(root, path, new_key, &item_size,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004528 item_size, item_size +
4529 sizeof(struct btrfs_item), 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004530 leaf = path->nodes[0];
4531 memcpy_extent_buffer(leaf,
4532 btrfs_item_ptr_offset(leaf, path->slots[0]),
4533 btrfs_item_ptr_offset(leaf, path->slots[0] - 1),
4534 item_size);
4535 return 0;
4536}
4537
4538/*
Chris Masond352ac62008-09-29 15:18:18 -04004539 * make the item pointed to by the path smaller. new_size indicates
4540 * how small to make it, and from_end tells us if we just chop bytes
4541 * off the end of the item or if we shift the item to chop bytes off
4542 * the front.
4543 */
David Sterba78ac4f92019-03-20 14:49:12 +01004544void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end)
Chris Masonb18c6682007-04-17 13:26:50 -04004545{
Chris Masonb18c6682007-04-17 13:26:50 -04004546 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004547 struct extent_buffer *leaf;
4548 struct btrfs_item *item;
Chris Masonb18c6682007-04-17 13:26:50 -04004549 u32 nritems;
4550 unsigned int data_end;
4551 unsigned int old_data_start;
4552 unsigned int old_size;
4553 unsigned int size_diff;
4554 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004555 struct btrfs_map_token token;
4556
4557 btrfs_init_map_token(&token);
Chris Masonb18c6682007-04-17 13:26:50 -04004558
Chris Mason5f39d392007-10-15 16:14:19 -04004559 leaf = path->nodes[0];
Chris Mason179e29e2007-11-01 11:28:41 -04004560 slot = path->slots[0];
4561
4562 old_size = btrfs_item_size_nr(leaf, slot);
4563 if (old_size == new_size)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004564 return;
Chris Masonb18c6682007-04-17 13:26:50 -04004565
Chris Mason5f39d392007-10-15 16:14:19 -04004566 nritems = btrfs_header_nritems(leaf);
David Sterba8f881e82019-03-20 11:33:10 +01004567 data_end = leaf_data_end(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004568
Chris Mason5f39d392007-10-15 16:14:19 -04004569 old_data_start = btrfs_item_offset_nr(leaf, slot);
Chris Mason179e29e2007-11-01 11:28:41 -04004570
Chris Masonb18c6682007-04-17 13:26:50 -04004571 size_diff = old_size - new_size;
4572
4573 BUG_ON(slot < 0);
4574 BUG_ON(slot >= nritems);
4575
4576 /*
4577 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4578 */
4579 /* first correct the data pointers */
4580 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004581 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004582 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004583
Chris Masoncfed81a2012-03-03 07:40:03 -05004584 ioff = btrfs_token_item_offset(leaf, item, &token);
4585 btrfs_set_token_item_offset(leaf, item,
4586 ioff + size_diff, &token);
Chris Masonb18c6682007-04-17 13:26:50 -04004587 }
Chris Masondb945352007-10-15 16:15:53 -04004588
Chris Masonb18c6682007-04-17 13:26:50 -04004589 /* shift the data */
Chris Mason179e29e2007-11-01 11:28:41 -04004590 if (from_end) {
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03004591 memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
4592 data_end + size_diff, BTRFS_LEAF_DATA_OFFSET +
Chris Mason179e29e2007-11-01 11:28:41 -04004593 data_end, old_data_start + new_size - data_end);
4594 } else {
4595 struct btrfs_disk_key disk_key;
4596 u64 offset;
4597
4598 btrfs_item_key(leaf, &disk_key, slot);
4599
4600 if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) {
4601 unsigned long ptr;
4602 struct btrfs_file_extent_item *fi;
4603
4604 fi = btrfs_item_ptr(leaf, slot,
4605 struct btrfs_file_extent_item);
4606 fi = (struct btrfs_file_extent_item *)(
4607 (unsigned long)fi - size_diff);
4608
4609 if (btrfs_file_extent_type(leaf, fi) ==
4610 BTRFS_FILE_EXTENT_INLINE) {
4611 ptr = btrfs_item_ptr_offset(leaf, slot);
4612 memmove_extent_buffer(leaf, ptr,
Chris Masond3977122009-01-05 21:25:51 -05004613 (unsigned long)fi,
David Sterba7ec20af2014-07-24 17:34:58 +02004614 BTRFS_FILE_EXTENT_INLINE_DATA_START);
Chris Mason179e29e2007-11-01 11:28:41 -04004615 }
4616 }
4617
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03004618 memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
4619 data_end + size_diff, BTRFS_LEAF_DATA_OFFSET +
Chris Mason179e29e2007-11-01 11:28:41 -04004620 data_end, old_data_start - data_end);
4621
4622 offset = btrfs_disk_key_offset(&disk_key);
4623 btrfs_set_disk_key_offset(&disk_key, offset + size_diff);
4624 btrfs_set_item_key(leaf, &disk_key, slot);
4625 if (slot == 0)
Nikolay Borisovb167fa92018-06-20 15:48:47 +03004626 fixup_low_keys(path, &disk_key, 1);
Chris Mason179e29e2007-11-01 11:28:41 -04004627 }
Chris Mason5f39d392007-10-15 16:14:19 -04004628
Ross Kirkdd3cc162013-09-16 15:58:09 +01004629 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004630 btrfs_set_item_size(leaf, item, new_size);
4631 btrfs_mark_buffer_dirty(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004632
David Sterbae902baa2019-03-20 14:36:46 +01004633 if (btrfs_leaf_free_space(leaf) < 0) {
David Sterbaa4f78752017-06-29 18:37:49 +02004634 btrfs_print_leaf(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004635 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004636 }
Chris Masonb18c6682007-04-17 13:26:50 -04004637}
4638
Chris Masond352ac62008-09-29 15:18:18 -04004639/*
Stefan Behrens8f69dbd2013-05-07 10:23:30 +00004640 * make the item pointed to by the path bigger, data_size is the added size.
Chris Masond352ac62008-09-29 15:18:18 -04004641 */
David Sterbac71dd882019-03-20 14:51:10 +01004642void btrfs_extend_item(struct btrfs_path *path, u32 data_size)
Chris Mason6567e832007-04-16 09:22:45 -04004643{
Chris Mason6567e832007-04-16 09:22:45 -04004644 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004645 struct extent_buffer *leaf;
4646 struct btrfs_item *item;
Chris Mason6567e832007-04-16 09:22:45 -04004647 u32 nritems;
4648 unsigned int data_end;
4649 unsigned int old_data;
4650 unsigned int old_size;
4651 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004652 struct btrfs_map_token token;
4653
4654 btrfs_init_map_token(&token);
Chris Mason6567e832007-04-16 09:22:45 -04004655
Chris Mason5f39d392007-10-15 16:14:19 -04004656 leaf = path->nodes[0];
Chris Mason6567e832007-04-16 09:22:45 -04004657
Chris Mason5f39d392007-10-15 16:14:19 -04004658 nritems = btrfs_header_nritems(leaf);
David Sterba8f881e82019-03-20 11:33:10 +01004659 data_end = leaf_data_end(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004660
David Sterbae902baa2019-03-20 14:36:46 +01004661 if (btrfs_leaf_free_space(leaf) < data_size) {
David Sterbaa4f78752017-06-29 18:37:49 +02004662 btrfs_print_leaf(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004663 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004664 }
Chris Mason6567e832007-04-16 09:22:45 -04004665 slot = path->slots[0];
Chris Mason5f39d392007-10-15 16:14:19 -04004666 old_data = btrfs_item_end_nr(leaf, slot);
Chris Mason6567e832007-04-16 09:22:45 -04004667
4668 BUG_ON(slot < 0);
Chris Mason3326d1b2007-10-15 16:18:25 -04004669 if (slot >= nritems) {
David Sterbaa4f78752017-06-29 18:37:49 +02004670 btrfs_print_leaf(leaf);
David Sterbac71dd882019-03-20 14:51:10 +01004671 btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d",
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004672 slot, nritems);
Arnd Bergmann290342f2019-03-25 14:02:25 +01004673 BUG();
Chris Mason3326d1b2007-10-15 16:18:25 -04004674 }
Chris Mason6567e832007-04-16 09:22:45 -04004675
4676 /*
4677 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4678 */
4679 /* first correct the data pointers */
4680 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004681 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004682 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004683
Chris Masoncfed81a2012-03-03 07:40:03 -05004684 ioff = btrfs_token_item_offset(leaf, item, &token);
4685 btrfs_set_token_item_offset(leaf, item,
4686 ioff - data_size, &token);
Chris Mason6567e832007-04-16 09:22:45 -04004687 }
Chris Mason5f39d392007-10-15 16:14:19 -04004688
Chris Mason6567e832007-04-16 09:22:45 -04004689 /* shift the data */
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03004690 memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
4691 data_end - data_size, BTRFS_LEAF_DATA_OFFSET +
Chris Mason6567e832007-04-16 09:22:45 -04004692 data_end, old_data - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004693
Chris Mason6567e832007-04-16 09:22:45 -04004694 data_end = old_data;
Chris Mason5f39d392007-10-15 16:14:19 -04004695 old_size = btrfs_item_size_nr(leaf, slot);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004696 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004697 btrfs_set_item_size(leaf, item, old_size + data_size);
4698 btrfs_mark_buffer_dirty(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004699
David Sterbae902baa2019-03-20 14:36:46 +01004700 if (btrfs_leaf_free_space(leaf) < 0) {
David Sterbaa4f78752017-06-29 18:37:49 +02004701 btrfs_print_leaf(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004702 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004703 }
Chris Mason6567e832007-04-16 09:22:45 -04004704}
4705
Chris Mason74123bd2007-02-02 11:05:29 -05004706/*
Chris Mason44871b12009-03-13 10:04:31 -04004707 * this is a helper for btrfs_insert_empty_items, the main goal here is
4708 * to save stack depth by doing the bulk of the work in a function
4709 * that doesn't call btrfs_search_slot
Chris Mason74123bd2007-02-02 11:05:29 -05004710 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004711void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path,
Omar Sandoval310712b2017-01-17 23:24:37 -08004712 const struct btrfs_key *cpu_key, u32 *data_size,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004713 u32 total_data, u32 total_size, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004714{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004715 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04004716 struct btrfs_item *item;
Chris Mason9c583092008-01-29 15:15:18 -05004717 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004718 u32 nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004719 unsigned int data_end;
Chris Masone2fa7222007-03-12 16:22:34 -04004720 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004721 struct extent_buffer *leaf;
4722 int slot;
Chris Masoncfed81a2012-03-03 07:40:03 -05004723 struct btrfs_map_token token;
4724
Filipe Manana24cdc842014-07-28 19:34:35 +01004725 if (path->slots[0] == 0) {
4726 btrfs_cpu_key_to_disk(&disk_key, cpu_key);
Nikolay Borisovb167fa92018-06-20 15:48:47 +03004727 fixup_low_keys(path, &disk_key, 1);
Filipe Manana24cdc842014-07-28 19:34:35 +01004728 }
4729 btrfs_unlock_up_safe(path, 1);
4730
Chris Masoncfed81a2012-03-03 07:40:03 -05004731 btrfs_init_map_token(&token);
Chris Masone2fa7222007-03-12 16:22:34 -04004732
Chris Mason5f39d392007-10-15 16:14:19 -04004733 leaf = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04004734 slot = path->slots[0];
Chris Mason74123bd2007-02-02 11:05:29 -05004735
Chris Mason5f39d392007-10-15 16:14:19 -04004736 nritems = btrfs_header_nritems(leaf);
David Sterba8f881e82019-03-20 11:33:10 +01004737 data_end = leaf_data_end(leaf);
Chris Masoneb60cea2007-02-02 09:18:22 -05004738
David Sterbae902baa2019-03-20 14:36:46 +01004739 if (btrfs_leaf_free_space(leaf) < total_size) {
David Sterbaa4f78752017-06-29 18:37:49 +02004740 btrfs_print_leaf(leaf);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004741 btrfs_crit(fs_info, "not enough freespace need %u have %d",
David Sterbae902baa2019-03-20 14:36:46 +01004742 total_size, btrfs_leaf_free_space(leaf));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004743 BUG();
Chris Masond4dbff92007-04-04 14:08:15 -04004744 }
Chris Mason5f39d392007-10-15 16:14:19 -04004745
Chris Masonbe0e5c02007-01-26 15:51:26 -05004746 if (slot != nritems) {
Chris Mason5f39d392007-10-15 16:14:19 -04004747 unsigned int old_data = btrfs_item_end_nr(leaf, slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004748
Chris Mason5f39d392007-10-15 16:14:19 -04004749 if (old_data < data_end) {
David Sterbaa4f78752017-06-29 18:37:49 +02004750 btrfs_print_leaf(leaf);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004751 btrfs_crit(fs_info, "slot %d old_data %d data_end %d",
Jeff Mahoney5d163e02016-09-20 10:05:00 -04004752 slot, old_data, data_end);
Arnd Bergmann290342f2019-03-25 14:02:25 +01004753 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004754 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004755 /*
4756 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4757 */
4758 /* first correct the data pointers */
Chris Mason0783fcf2007-03-12 20:12:07 -04004759 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004760 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004761
Jeff Mahoney62e85572016-09-20 10:05:01 -04004762 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004763 ioff = btrfs_token_item_offset(leaf, item, &token);
4764 btrfs_set_token_item_offset(leaf, item,
4765 ioff - total_data, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004766 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004767 /* shift the items */
Chris Mason9c583092008-01-29 15:15:18 -05004768 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
Chris Mason5f39d392007-10-15 16:14:19 -04004769 btrfs_item_nr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04004770 (nritems - slot) * sizeof(struct btrfs_item));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004771
4772 /* shift the data */
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03004773 memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
4774 data_end - total_data, BTRFS_LEAF_DATA_OFFSET +
Chris Masond6025572007-03-30 14:27:56 -04004775 data_end, old_data - data_end);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004776 data_end = old_data;
4777 }
Chris Mason5f39d392007-10-15 16:14:19 -04004778
Chris Mason62e27492007-03-15 12:56:47 -04004779 /* setup the item for the new data */
Chris Mason9c583092008-01-29 15:15:18 -05004780 for (i = 0; i < nr; i++) {
4781 btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
4782 btrfs_set_item_key(leaf, &disk_key, slot + i);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004783 item = btrfs_item_nr(slot + i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004784 btrfs_set_token_item_offset(leaf, item,
4785 data_end - data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004786 data_end -= data_size[i];
Chris Masoncfed81a2012-03-03 07:40:03 -05004787 btrfs_set_token_item_size(leaf, item, data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004788 }
Chris Mason44871b12009-03-13 10:04:31 -04004789
Chris Mason9c583092008-01-29 15:15:18 -05004790 btrfs_set_header_nritems(leaf, nritems + nr);
Chris Masonb9473432009-03-13 11:00:37 -04004791 btrfs_mark_buffer_dirty(leaf);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004792
David Sterbae902baa2019-03-20 14:36:46 +01004793 if (btrfs_leaf_free_space(leaf) < 0) {
David Sterbaa4f78752017-06-29 18:37:49 +02004794 btrfs_print_leaf(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004795 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004796 }
Chris Mason44871b12009-03-13 10:04:31 -04004797}
4798
4799/*
4800 * Given a key and some data, insert items into the tree.
4801 * This does all the path init required, making room in the tree if needed.
4802 */
4803int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
4804 struct btrfs_root *root,
4805 struct btrfs_path *path,
Omar Sandoval310712b2017-01-17 23:24:37 -08004806 const struct btrfs_key *cpu_key, u32 *data_size,
Chris Mason44871b12009-03-13 10:04:31 -04004807 int nr)
4808{
Chris Mason44871b12009-03-13 10:04:31 -04004809 int ret = 0;
4810 int slot;
4811 int i;
4812 u32 total_size = 0;
4813 u32 total_data = 0;
4814
4815 for (i = 0; i < nr; i++)
4816 total_data += data_size[i];
4817
4818 total_size = total_data + (nr * sizeof(struct btrfs_item));
4819 ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
4820 if (ret == 0)
4821 return -EEXIST;
4822 if (ret < 0)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004823 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004824
Chris Mason44871b12009-03-13 10:04:31 -04004825 slot = path->slots[0];
4826 BUG_ON(slot < 0);
4827
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004828 setup_items_for_insert(root, path, cpu_key, data_size,
Chris Mason44871b12009-03-13 10:04:31 -04004829 total_data, total_size, nr);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004830 return 0;
Chris Mason62e27492007-03-15 12:56:47 -04004831}
4832
4833/*
4834 * Given a key and some data, insert an item into the tree.
4835 * This does all the path init required, making room in the tree if needed.
4836 */
Omar Sandoval310712b2017-01-17 23:24:37 -08004837int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4838 const struct btrfs_key *cpu_key, void *data,
4839 u32 data_size)
Chris Mason62e27492007-03-15 12:56:47 -04004840{
4841 int ret = 0;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004842 struct btrfs_path *path;
Chris Mason5f39d392007-10-15 16:14:19 -04004843 struct extent_buffer *leaf;
4844 unsigned long ptr;
Chris Mason62e27492007-03-15 12:56:47 -04004845
Chris Mason2c90e5d2007-04-02 10:50:19 -04004846 path = btrfs_alloc_path();
Tsutomu Itohdb5b4932011-03-23 08:14:16 +00004847 if (!path)
4848 return -ENOMEM;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004849 ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size);
Chris Mason62e27492007-03-15 12:56:47 -04004850 if (!ret) {
Chris Mason5f39d392007-10-15 16:14:19 -04004851 leaf = path->nodes[0];
4852 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
4853 write_extent_buffer(leaf, data, ptr, data_size);
4854 btrfs_mark_buffer_dirty(leaf);
Chris Mason62e27492007-03-15 12:56:47 -04004855 }
Chris Mason2c90e5d2007-04-02 10:50:19 -04004856 btrfs_free_path(path);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004857 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004858}
4859
Chris Mason74123bd2007-02-02 11:05:29 -05004860/*
Chris Mason5de08d72007-02-24 06:24:44 -05004861 * delete the pointer from a given node.
Chris Mason74123bd2007-02-02 11:05:29 -05004862 *
Chris Masond352ac62008-09-29 15:18:18 -04004863 * the tree should have been previously balanced so the deletion does not
4864 * empty a node.
Chris Mason74123bd2007-02-02 11:05:29 -05004865 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004866static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
4867 int level, int slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004868{
Chris Mason5f39d392007-10-15 16:14:19 -04004869 struct extent_buffer *parent = path->nodes[level];
Chris Mason7518a232007-03-12 12:01:18 -04004870 u32 nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004871 int ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004872
Chris Mason5f39d392007-10-15 16:14:19 -04004873 nritems = btrfs_header_nritems(parent);
Chris Masond3977122009-01-05 21:25:51 -05004874 if (slot != nritems - 1) {
David Sterbabf1d3422018-03-05 15:47:39 +01004875 if (level) {
4876 ret = tree_mod_log_insert_move(parent, slot, slot + 1,
David Sterbaa446a972018-03-05 15:26:29 +01004877 nritems - slot - 1);
David Sterbabf1d3422018-03-05 15:47:39 +01004878 BUG_ON(ret < 0);
4879 }
Chris Mason5f39d392007-10-15 16:14:19 -04004880 memmove_extent_buffer(parent,
4881 btrfs_node_key_ptr_offset(slot),
4882 btrfs_node_key_ptr_offset(slot + 1),
Chris Masond6025572007-03-30 14:27:56 -04004883 sizeof(struct btrfs_key_ptr) *
4884 (nritems - slot - 1));
Chris Mason57ba86c2012-12-18 19:35:32 -05004885 } else if (level) {
David Sterbae09c2ef2018-03-05 15:09:03 +01004886 ret = tree_mod_log_insert_key(parent, slot, MOD_LOG_KEY_REMOVE,
4887 GFP_NOFS);
Chris Mason57ba86c2012-12-18 19:35:32 -05004888 BUG_ON(ret < 0);
Chris Masonbb803952007-03-01 12:04:21 -05004889 }
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004890
Chris Mason7518a232007-03-12 12:01:18 -04004891 nritems--;
Chris Mason5f39d392007-10-15 16:14:19 -04004892 btrfs_set_header_nritems(parent, nritems);
Chris Mason7518a232007-03-12 12:01:18 -04004893 if (nritems == 0 && parent == root->node) {
Chris Mason5f39d392007-10-15 16:14:19 -04004894 BUG_ON(btrfs_header_level(root->node) != 1);
Chris Masonbb803952007-03-01 12:04:21 -05004895 /* just turn the root into a leaf and break */
Chris Mason5f39d392007-10-15 16:14:19 -04004896 btrfs_set_header_level(root->node, 0);
Chris Masonbb803952007-03-01 12:04:21 -05004897 } else if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004898 struct btrfs_disk_key disk_key;
4899
4900 btrfs_node_key(parent, &disk_key, 0);
Nikolay Borisovb167fa92018-06-20 15:48:47 +03004901 fixup_low_keys(path, &disk_key, level + 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004902 }
Chris Masond6025572007-03-30 14:27:56 -04004903 btrfs_mark_buffer_dirty(parent);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004904}
4905
Chris Mason74123bd2007-02-02 11:05:29 -05004906/*
Chris Mason323ac952008-10-01 19:05:46 -04004907 * a helper function to delete the leaf pointed to by path->slots[1] and
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004908 * path->nodes[1].
Chris Mason323ac952008-10-01 19:05:46 -04004909 *
4910 * This deletes the pointer in path->nodes[1] and frees the leaf
4911 * block extent. zero is returned if it all worked out, < 0 otherwise.
4912 *
4913 * The path must have already been setup for deleting the leaf, including
4914 * all the proper balancing. path->nodes[1] must be locked.
4915 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004916static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans,
4917 struct btrfs_root *root,
4918 struct btrfs_path *path,
4919 struct extent_buffer *leaf)
Chris Mason323ac952008-10-01 19:05:46 -04004920{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004921 WARN_ON(btrfs_header_generation(leaf) != trans->transid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004922 del_ptr(root, path, 1, path->slots[1]);
Chris Mason323ac952008-10-01 19:05:46 -04004923
Chris Mason4d081c42009-02-04 09:31:28 -05004924 /*
4925 * btrfs_free_extent is expensive, we want to make sure we
4926 * aren't holding any locks when we call it
4927 */
4928 btrfs_unlock_up_safe(path, 0);
4929
Yan, Zhengf0486c62010-05-16 10:46:25 -04004930 root_sub_used(root, leaf->len);
4931
Josef Bacik3083ee22012-03-09 16:01:49 -05004932 extent_buffer_get(leaf);
Jan Schmidt5581a512012-05-16 17:04:52 +02004933 btrfs_free_tree_block(trans, root, leaf, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05004934 free_extent_buffer_stale(leaf);
Chris Mason323ac952008-10-01 19:05:46 -04004935}
4936/*
Chris Mason74123bd2007-02-02 11:05:29 -05004937 * delete the item at the leaf level in path. If that empties
4938 * the leaf, remove it from the tree
4939 */
Chris Mason85e21ba2008-01-29 15:11:36 -05004940int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4941 struct btrfs_path *path, int slot, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004942{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004943 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason5f39d392007-10-15 16:14:19 -04004944 struct extent_buffer *leaf;
4945 struct btrfs_item *item;
Alexandru Moisece0eac22015-08-23 16:01:42 +00004946 u32 last_off;
4947 u32 dsize = 0;
Chris Masonaa5d6be2007-02-28 16:35:06 -05004948 int ret = 0;
4949 int wret;
Chris Mason85e21ba2008-01-29 15:11:36 -05004950 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004951 u32 nritems;
Chris Masoncfed81a2012-03-03 07:40:03 -05004952 struct btrfs_map_token token;
4953
4954 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004955
Chris Mason5f39d392007-10-15 16:14:19 -04004956 leaf = path->nodes[0];
Chris Mason85e21ba2008-01-29 15:11:36 -05004957 last_off = btrfs_item_offset_nr(leaf, slot + nr - 1);
4958
4959 for (i = 0; i < nr; i++)
4960 dsize += btrfs_item_size_nr(leaf, slot + i);
4961
Chris Mason5f39d392007-10-15 16:14:19 -04004962 nritems = btrfs_header_nritems(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004963
Chris Mason85e21ba2008-01-29 15:11:36 -05004964 if (slot + nr != nritems) {
David Sterba8f881e82019-03-20 11:33:10 +01004965 int data_end = leaf_data_end(leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04004966
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03004967 memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
Chris Masond6025572007-03-30 14:27:56 -04004968 data_end + dsize,
Nikolay Borisov3d9ec8c2017-05-29 09:43:43 +03004969 BTRFS_LEAF_DATA_OFFSET + data_end,
Chris Mason85e21ba2008-01-29 15:11:36 -05004970 last_off - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004971
Chris Mason85e21ba2008-01-29 15:11:36 -05004972 for (i = slot + nr; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004973 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004974
Ross Kirkdd3cc162013-09-16 15:58:09 +01004975 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004976 ioff = btrfs_token_item_offset(leaf, item, &token);
4977 btrfs_set_token_item_offset(leaf, item,
4978 ioff + dsize, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004979 }
Chris Masondb945352007-10-15 16:15:53 -04004980
Chris Mason5f39d392007-10-15 16:14:19 -04004981 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),
Chris Mason85e21ba2008-01-29 15:11:36 -05004982 btrfs_item_nr_offset(slot + nr),
Chris Masond6025572007-03-30 14:27:56 -04004983 sizeof(struct btrfs_item) *
Chris Mason85e21ba2008-01-29 15:11:36 -05004984 (nritems - slot - nr));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004985 }
Chris Mason85e21ba2008-01-29 15:11:36 -05004986 btrfs_set_header_nritems(leaf, nritems - nr);
4987 nritems -= nr;
Chris Mason5f39d392007-10-15 16:14:19 -04004988
Chris Mason74123bd2007-02-02 11:05:29 -05004989 /* delete the leaf if we've emptied it */
Chris Mason7518a232007-03-12 12:01:18 -04004990 if (nritems == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004991 if (leaf == root->node) {
4992 btrfs_set_header_level(leaf, 0);
Chris Mason9a8dd152007-02-23 08:38:36 -05004993 } else {
Yan, Zhengf0486c62010-05-16 10:46:25 -04004994 btrfs_set_path_blocking(path);
David Sterba6a884d7d2019-03-20 14:30:02 +01004995 btrfs_clean_tree_block(leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004996 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason9a8dd152007-02-23 08:38:36 -05004997 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004998 } else {
Chris Mason7518a232007-03-12 12:01:18 -04004999 int used = leaf_space_used(leaf, 0, nritems);
Chris Masonaa5d6be2007-02-28 16:35:06 -05005000 if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04005001 struct btrfs_disk_key disk_key;
5002
5003 btrfs_item_key(leaf, &disk_key, 0);
Nikolay Borisovb167fa92018-06-20 15:48:47 +03005004 fixup_low_keys(path, &disk_key, 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05005005 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05005006
Chris Mason74123bd2007-02-02 11:05:29 -05005007 /* delete the leaf if it is mostly empty */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005008 if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05005009 /* push_leaf_left fixes the path.
5010 * make sure the path still points to our leaf
5011 * for possible call to del_ptr below
5012 */
Chris Mason4920c9a2007-01-26 16:38:42 -05005013 slot = path->slots[1];
Chris Mason5f39d392007-10-15 16:14:19 -04005014 extent_buffer_get(leaf);
5015
Chris Masonb9473432009-03-13 11:00:37 -04005016 btrfs_set_path_blocking(path);
Chris Mason99d8f832010-07-07 10:51:48 -04005017 wret = push_leaf_left(trans, root, path, 1, 1,
5018 1, (u32)-1);
Chris Mason54aa1f42007-06-22 14:16:25 -04005019 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005020 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04005021
5022 if (path->nodes[0] == leaf &&
5023 btrfs_header_nritems(leaf)) {
Chris Mason99d8f832010-07-07 10:51:48 -04005024 wret = push_leaf_right(trans, root, path, 1,
5025 1, 1, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04005026 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005027 ret = wret;
5028 }
Chris Mason5f39d392007-10-15 16:14:19 -04005029
5030 if (btrfs_header_nritems(leaf) == 0) {
Chris Mason323ac952008-10-01 19:05:46 -04005031 path->slots[1] = slot;
Jeff Mahoney143bede2012-03-01 14:56:26 +01005032 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005033 free_extent_buffer(leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01005034 ret = 0;
Chris Mason5de08d72007-02-24 06:24:44 -05005035 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005036 /* if we're still in the path, make sure
5037 * we're dirty. Otherwise, one of the
5038 * push_leaf functions must have already
5039 * dirtied this buffer
5040 */
5041 if (path->nodes[0] == leaf)
5042 btrfs_mark_buffer_dirty(leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005043 free_extent_buffer(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005044 }
Chris Masond5719762007-03-23 10:01:08 -04005045 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04005046 btrfs_mark_buffer_dirty(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005047 }
5048 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05005049 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05005050}
5051
Chris Mason97571fd2007-02-24 13:39:08 -05005052/*
Chris Mason925baed2008-06-25 16:01:30 -04005053 * search the tree again to find a leaf with lesser keys
Chris Mason7bb86312007-12-11 09:25:06 -05005054 * returns 0 if it found something or 1 if there are no lesser leaves.
5055 * returns < 0 on io errors.
Chris Masond352ac62008-09-29 15:18:18 -04005056 *
5057 * This may release the path, and so you may lose any locks held at the
5058 * time you call it.
Chris Mason7bb86312007-12-11 09:25:06 -05005059 */
Josef Bacik16e75492013-10-22 12:18:51 -04005060int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Mason7bb86312007-12-11 09:25:06 -05005061{
Chris Mason925baed2008-06-25 16:01:30 -04005062 struct btrfs_key key;
5063 struct btrfs_disk_key found_key;
5064 int ret;
Chris Mason7bb86312007-12-11 09:25:06 -05005065
Chris Mason925baed2008-06-25 16:01:30 -04005066 btrfs_item_key_to_cpu(path->nodes[0], &key, 0);
Chris Mason7bb86312007-12-11 09:25:06 -05005067
Filipe David Borba Mananae8b0d7242013-10-15 00:12:27 +01005068 if (key.offset > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005069 key.offset--;
Filipe David Borba Mananae8b0d7242013-10-15 00:12:27 +01005070 } else if (key.type > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005071 key.type--;
Filipe David Borba Mananae8b0d7242013-10-15 00:12:27 +01005072 key.offset = (u64)-1;
5073 } else if (key.objectid > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005074 key.objectid--;
Filipe David Borba Mananae8b0d7242013-10-15 00:12:27 +01005075 key.type = (u8)-1;
5076 key.offset = (u64)-1;
5077 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005078 return 1;
Filipe David Borba Mananae8b0d7242013-10-15 00:12:27 +01005079 }
Chris Mason7bb86312007-12-11 09:25:06 -05005080
David Sterbab3b4aa72011-04-21 01:20:15 +02005081 btrfs_release_path(path);
Chris Mason925baed2008-06-25 16:01:30 -04005082 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5083 if (ret < 0)
5084 return ret;
5085 btrfs_item_key(path->nodes[0], &found_key, 0);
5086 ret = comp_keys(&found_key, &key);
Filipe Manana337c6f62014-06-09 13:22:13 +01005087 /*
5088 * We might have had an item with the previous key in the tree right
5089 * before we released our path. And after we released our path, that
5090 * item might have been pushed to the first slot (0) of the leaf we
5091 * were holding due to a tree balance. Alternatively, an item with the
5092 * previous key can exist as the only element of a leaf (big fat item).
5093 * Therefore account for these 2 cases, so that our callers (like
5094 * btrfs_previous_item) don't miss an existing item with a key matching
5095 * the previous key we computed above.
5096 */
5097 if (ret <= 0)
Chris Mason925baed2008-06-25 16:01:30 -04005098 return 0;
5099 return 1;
Chris Mason7bb86312007-12-11 09:25:06 -05005100}
5101
Chris Mason3f157a22008-06-25 16:01:31 -04005102/*
5103 * A helper function to walk down the tree starting at min_key, and looking
Eric Sandeende78b512013-01-31 18:21:12 +00005104 * for nodes or leaves that are have a minimum transaction id.
5105 * This is used by the btree defrag code, and tree logging
Chris Mason3f157a22008-06-25 16:01:31 -04005106 *
5107 * This does not cow, but it does stuff the starting key it finds back
5108 * into min_key, so you can call btrfs_search_slot with cow=1 on the
5109 * key and get a writable path.
5110 *
Chris Mason3f157a22008-06-25 16:01:31 -04005111 * This honors path->lowest_level to prevent descent past a given level
5112 * of the tree.
5113 *
Chris Masond352ac62008-09-29 15:18:18 -04005114 * min_trans indicates the oldest transaction that you are interested
5115 * in walking through. Any nodes or leaves older than min_trans are
5116 * skipped over (without reading them).
5117 *
Chris Mason3f157a22008-06-25 16:01:31 -04005118 * returns zero if something useful was found, < 0 on error and 1 if there
5119 * was nothing in the tree that matched the search criteria.
5120 */
5121int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
Eric Sandeende78b512013-01-31 18:21:12 +00005122 struct btrfs_path *path,
Chris Mason3f157a22008-06-25 16:01:31 -04005123 u64 min_trans)
5124{
5125 struct extent_buffer *cur;
5126 struct btrfs_key found_key;
5127 int slot;
Yan96524802008-07-24 12:19:49 -04005128 int sret;
Chris Mason3f157a22008-06-25 16:01:31 -04005129 u32 nritems;
5130 int level;
5131 int ret = 1;
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005132 int keep_locks = path->keep_locks;
Chris Mason3f157a22008-06-25 16:01:31 -04005133
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005134 path->keep_locks = 1;
Chris Mason3f157a22008-06-25 16:01:31 -04005135again:
Chris Masonbd681512011-07-16 15:23:14 -04005136 cur = btrfs_read_lock_root_node(root);
Chris Mason3f157a22008-06-25 16:01:31 -04005137 level = btrfs_header_level(cur);
Chris Masone02119d2008-09-05 16:13:11 -04005138 WARN_ON(path->nodes[level]);
Chris Mason3f157a22008-06-25 16:01:31 -04005139 path->nodes[level] = cur;
Chris Masonbd681512011-07-16 15:23:14 -04005140 path->locks[level] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005141
5142 if (btrfs_header_generation(cur) < min_trans) {
5143 ret = 1;
5144 goto out;
5145 }
Chris Masond3977122009-01-05 21:25:51 -05005146 while (1) {
Chris Mason3f157a22008-06-25 16:01:31 -04005147 nritems = btrfs_header_nritems(cur);
5148 level = btrfs_header_level(cur);
Nikolay Borisova74b35e2017-12-08 16:27:43 +02005149 sret = btrfs_bin_search(cur, min_key, level, &slot);
Filipe Mananacbca7d52019-02-18 16:57:26 +00005150 if (sret < 0) {
5151 ret = sret;
5152 goto out;
5153 }
Chris Mason3f157a22008-06-25 16:01:31 -04005154
Chris Mason323ac952008-10-01 19:05:46 -04005155 /* at the lowest level, we're done, setup the path and exit */
5156 if (level == path->lowest_level) {
Chris Masone02119d2008-09-05 16:13:11 -04005157 if (slot >= nritems)
5158 goto find_next_key;
Chris Mason3f157a22008-06-25 16:01:31 -04005159 ret = 0;
5160 path->slots[level] = slot;
5161 btrfs_item_key_to_cpu(cur, &found_key, slot);
5162 goto out;
5163 }
Yan96524802008-07-24 12:19:49 -04005164 if (sret && slot > 0)
5165 slot--;
Chris Mason3f157a22008-06-25 16:01:31 -04005166 /*
Eric Sandeende78b512013-01-31 18:21:12 +00005167 * check this node pointer against the min_trans parameters.
5168 * If it is too old, old, skip to the next one.
Chris Mason3f157a22008-06-25 16:01:31 -04005169 */
Chris Masond3977122009-01-05 21:25:51 -05005170 while (slot < nritems) {
Chris Mason3f157a22008-06-25 16:01:31 -04005171 u64 gen;
Chris Masone02119d2008-09-05 16:13:11 -04005172
Chris Mason3f157a22008-06-25 16:01:31 -04005173 gen = btrfs_node_ptr_generation(cur, slot);
5174 if (gen < min_trans) {
5175 slot++;
5176 continue;
5177 }
Eric Sandeende78b512013-01-31 18:21:12 +00005178 break;
Chris Mason3f157a22008-06-25 16:01:31 -04005179 }
Chris Masone02119d2008-09-05 16:13:11 -04005180find_next_key:
Chris Mason3f157a22008-06-25 16:01:31 -04005181 /*
5182 * we didn't find a candidate key in this node, walk forward
5183 * and find another one
5184 */
5185 if (slot >= nritems) {
Chris Masone02119d2008-09-05 16:13:11 -04005186 path->slots[level] = slot;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005187 btrfs_set_path_blocking(path);
Chris Masone02119d2008-09-05 16:13:11 -04005188 sret = btrfs_find_next_key(root, path, min_key, level,
Eric Sandeende78b512013-01-31 18:21:12 +00005189 min_trans);
Chris Masone02119d2008-09-05 16:13:11 -04005190 if (sret == 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005191 btrfs_release_path(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005192 goto again;
5193 } else {
5194 goto out;
5195 }
5196 }
5197 /* save our key for returning back */
5198 btrfs_node_key_to_cpu(cur, &found_key, slot);
5199 path->slots[level] = slot;
5200 if (level == path->lowest_level) {
5201 ret = 0;
Chris Mason3f157a22008-06-25 16:01:31 -04005202 goto out;
5203 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05005204 btrfs_set_path_blocking(path);
David Sterbad0d20b02019-03-20 14:54:01 +01005205 cur = read_node_slot(cur, slot);
Liu Bofb770ae2016-07-05 12:10:14 -07005206 if (IS_ERR(cur)) {
5207 ret = PTR_ERR(cur);
5208 goto out;
5209 }
Chris Mason3f157a22008-06-25 16:01:31 -04005210
Chris Masonbd681512011-07-16 15:23:14 -04005211 btrfs_tree_read_lock(cur);
Chris Masonb4ce94d2009-02-04 09:25:08 -05005212
Chris Masonbd681512011-07-16 15:23:14 -04005213 path->locks[level - 1] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005214 path->nodes[level - 1] = cur;
Chris Masonf7c79f32012-03-19 15:54:38 -04005215 unlock_up(path, level, 1, 0, NULL);
Chris Mason3f157a22008-06-25 16:01:31 -04005216 }
5217out:
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005218 path->keep_locks = keep_locks;
5219 if (ret == 0) {
5220 btrfs_unlock_up_safe(path, path->lowest_level + 1);
5221 btrfs_set_path_blocking(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005222 memcpy(min_key, &found_key, sizeof(found_key));
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005223 }
Chris Mason3f157a22008-06-25 16:01:31 -04005224 return ret;
5225}
5226
David Sterbac7da9592019-03-20 15:02:46 +01005227static int tree_move_down(struct btrfs_path *path, int *level)
Alexander Block70698302012-06-05 21:07:48 +02005228{
Liu Bofb770ae2016-07-05 12:10:14 -07005229 struct extent_buffer *eb;
5230
Chris Mason74dd17f2012-08-07 16:25:13 -04005231 BUG_ON(*level == 0);
David Sterbad0d20b02019-03-20 14:54:01 +01005232 eb = read_node_slot(path->nodes[*level], path->slots[*level]);
Liu Bofb770ae2016-07-05 12:10:14 -07005233 if (IS_ERR(eb))
5234 return PTR_ERR(eb);
5235
5236 path->nodes[*level - 1] = eb;
Alexander Block70698302012-06-05 21:07:48 +02005237 path->slots[*level - 1] = 0;
5238 (*level)--;
Liu Bofb770ae2016-07-05 12:10:14 -07005239 return 0;
Alexander Block70698302012-06-05 21:07:48 +02005240}
5241
David Sterbaf1e30262017-02-10 19:25:51 +01005242static int tree_move_next_or_upnext(struct btrfs_path *path,
Alexander Block70698302012-06-05 21:07:48 +02005243 int *level, int root_level)
5244{
5245 int ret = 0;
5246 int nritems;
5247 nritems = btrfs_header_nritems(path->nodes[*level]);
5248
5249 path->slots[*level]++;
5250
Chris Mason74dd17f2012-08-07 16:25:13 -04005251 while (path->slots[*level] >= nritems) {
Alexander Block70698302012-06-05 21:07:48 +02005252 if (*level == root_level)
5253 return -1;
5254
5255 /* move upnext */
5256 path->slots[*level] = 0;
5257 free_extent_buffer(path->nodes[*level]);
5258 path->nodes[*level] = NULL;
5259 (*level)++;
5260 path->slots[*level]++;
5261
5262 nritems = btrfs_header_nritems(path->nodes[*level]);
5263 ret = 1;
5264 }
5265 return ret;
5266}
5267
5268/*
5269 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
5270 * or down.
5271 */
David Sterba179d1e62019-03-20 15:03:48 +01005272static int tree_advance(struct btrfs_path *path,
Alexander Block70698302012-06-05 21:07:48 +02005273 int *level, int root_level,
5274 int allow_down,
5275 struct btrfs_key *key)
5276{
5277 int ret;
5278
5279 if (*level == 0 || !allow_down) {
David Sterbaf1e30262017-02-10 19:25:51 +01005280 ret = tree_move_next_or_upnext(path, level, root_level);
Alexander Block70698302012-06-05 21:07:48 +02005281 } else {
David Sterbac7da9592019-03-20 15:02:46 +01005282 ret = tree_move_down(path, level);
Alexander Block70698302012-06-05 21:07:48 +02005283 }
5284 if (ret >= 0) {
5285 if (*level == 0)
5286 btrfs_item_key_to_cpu(path->nodes[*level], key,
5287 path->slots[*level]);
5288 else
5289 btrfs_node_key_to_cpu(path->nodes[*level], key,
5290 path->slots[*level]);
5291 }
5292 return ret;
5293}
5294
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04005295static int tree_compare_item(struct btrfs_path *left_path,
Alexander Block70698302012-06-05 21:07:48 +02005296 struct btrfs_path *right_path,
5297 char *tmp_buf)
5298{
5299 int cmp;
5300 int len1, len2;
5301 unsigned long off1, off2;
5302
5303 len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]);
5304 len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]);
5305 if (len1 != len2)
5306 return 1;
5307
5308 off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
5309 off2 = btrfs_item_ptr_offset(right_path->nodes[0],
5310 right_path->slots[0]);
5311
5312 read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
5313
5314 cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
5315 if (cmp)
5316 return 1;
5317 return 0;
5318}
5319
5320#define ADVANCE 1
5321#define ADVANCE_ONLY_NEXT -1
5322
5323/*
5324 * This function compares two trees and calls the provided callback for
5325 * every changed/new/deleted item it finds.
5326 * If shared tree blocks are encountered, whole subtrees are skipped, making
5327 * the compare pretty fast on snapshotted subvolumes.
5328 *
5329 * This currently works on commit roots only. As commit roots are read only,
5330 * we don't do any locking. The commit roots are protected with transactions.
5331 * Transactions are ended and rejoined when a commit is tried in between.
5332 *
5333 * This function checks for modifications done to the trees while comparing.
5334 * If it detects a change, it aborts immediately.
5335 */
5336int btrfs_compare_trees(struct btrfs_root *left_root,
5337 struct btrfs_root *right_root,
5338 btrfs_changed_cb_t changed_cb, void *ctx)
5339{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005340 struct btrfs_fs_info *fs_info = left_root->fs_info;
Alexander Block70698302012-06-05 21:07:48 +02005341 int ret;
5342 int cmp;
Alexander Block70698302012-06-05 21:07:48 +02005343 struct btrfs_path *left_path = NULL;
5344 struct btrfs_path *right_path = NULL;
5345 struct btrfs_key left_key;
5346 struct btrfs_key right_key;
5347 char *tmp_buf = NULL;
5348 int left_root_level;
5349 int right_root_level;
5350 int left_level;
5351 int right_level;
5352 int left_end_reached;
5353 int right_end_reached;
5354 int advance_left;
5355 int advance_right;
5356 u64 left_blockptr;
5357 u64 right_blockptr;
Filipe Manana6baa4292014-02-20 21:15:25 +00005358 u64 left_gen;
5359 u64 right_gen;
Alexander Block70698302012-06-05 21:07:48 +02005360
5361 left_path = btrfs_alloc_path();
5362 if (!left_path) {
5363 ret = -ENOMEM;
5364 goto out;
5365 }
5366 right_path = btrfs_alloc_path();
5367 if (!right_path) {
5368 ret = -ENOMEM;
5369 goto out;
5370 }
5371
Michal Hocko752ade62017-05-08 15:57:27 -07005372 tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
Alexander Block70698302012-06-05 21:07:48 +02005373 if (!tmp_buf) {
Michal Hocko752ade62017-05-08 15:57:27 -07005374 ret = -ENOMEM;
5375 goto out;
Alexander Block70698302012-06-05 21:07:48 +02005376 }
5377
5378 left_path->search_commit_root = 1;
5379 left_path->skip_locking = 1;
5380 right_path->search_commit_root = 1;
5381 right_path->skip_locking = 1;
5382
Alexander Block70698302012-06-05 21:07:48 +02005383 /*
5384 * Strategy: Go to the first items of both trees. Then do
5385 *
5386 * If both trees are at level 0
5387 * Compare keys of current items
5388 * If left < right treat left item as new, advance left tree
5389 * and repeat
5390 * If left > right treat right item as deleted, advance right tree
5391 * and repeat
5392 * If left == right do deep compare of items, treat as changed if
5393 * needed, advance both trees and repeat
5394 * If both trees are at the same level but not at level 0
5395 * Compare keys of current nodes/leafs
5396 * If left < right advance left tree and repeat
5397 * If left > right advance right tree and repeat
5398 * If left == right compare blockptrs of the next nodes/leafs
5399 * If they match advance both trees but stay at the same level
5400 * and repeat
5401 * If they don't match advance both trees while allowing to go
5402 * deeper and repeat
5403 * If tree levels are different
5404 * Advance the tree that needs it and repeat
5405 *
5406 * Advancing a tree means:
5407 * If we are at level 0, try to go to the next slot. If that's not
5408 * possible, go one level up and repeat. Stop when we found a level
5409 * where we could go to the next slot. We may at this point be on a
5410 * node or a leaf.
5411 *
5412 * If we are not at level 0 and not on shared tree blocks, go one
5413 * level deeper.
5414 *
5415 * If we are not at level 0 and on shared tree blocks, go one slot to
5416 * the right if possible or go up and right.
5417 */
5418
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005419 down_read(&fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005420 left_level = btrfs_header_level(left_root->commit_root);
5421 left_root_level = left_level;
Robbie Ko6f2f0b32018-05-14 10:51:34 +08005422 left_path->nodes[left_level] =
5423 btrfs_clone_extent_buffer(left_root->commit_root);
5424 if (!left_path->nodes[left_level]) {
5425 up_read(&fs_info->commit_root_sem);
5426 ret = -ENOMEM;
5427 goto out;
5428 }
Alexander Block70698302012-06-05 21:07:48 +02005429
5430 right_level = btrfs_header_level(right_root->commit_root);
5431 right_root_level = right_level;
Robbie Ko6f2f0b32018-05-14 10:51:34 +08005432 right_path->nodes[right_level] =
5433 btrfs_clone_extent_buffer(right_root->commit_root);
5434 if (!right_path->nodes[right_level]) {
5435 up_read(&fs_info->commit_root_sem);
5436 ret = -ENOMEM;
5437 goto out;
5438 }
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005439 up_read(&fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005440
5441 if (left_level == 0)
5442 btrfs_item_key_to_cpu(left_path->nodes[left_level],
5443 &left_key, left_path->slots[left_level]);
5444 else
5445 btrfs_node_key_to_cpu(left_path->nodes[left_level],
5446 &left_key, left_path->slots[left_level]);
5447 if (right_level == 0)
5448 btrfs_item_key_to_cpu(right_path->nodes[right_level],
5449 &right_key, right_path->slots[right_level]);
5450 else
5451 btrfs_node_key_to_cpu(right_path->nodes[right_level],
5452 &right_key, right_path->slots[right_level]);
5453
5454 left_end_reached = right_end_reached = 0;
5455 advance_left = advance_right = 0;
5456
5457 while (1) {
Alexander Block70698302012-06-05 21:07:48 +02005458 if (advance_left && !left_end_reached) {
David Sterba179d1e62019-03-20 15:03:48 +01005459 ret = tree_advance(left_path, &left_level,
Alexander Block70698302012-06-05 21:07:48 +02005460 left_root_level,
5461 advance_left != ADVANCE_ONLY_NEXT,
5462 &left_key);
Liu Bofb770ae2016-07-05 12:10:14 -07005463 if (ret == -1)
Alexander Block70698302012-06-05 21:07:48 +02005464 left_end_reached = ADVANCE;
Liu Bofb770ae2016-07-05 12:10:14 -07005465 else if (ret < 0)
5466 goto out;
Alexander Block70698302012-06-05 21:07:48 +02005467 advance_left = 0;
5468 }
5469 if (advance_right && !right_end_reached) {
David Sterba179d1e62019-03-20 15:03:48 +01005470 ret = tree_advance(right_path, &right_level,
Alexander Block70698302012-06-05 21:07:48 +02005471 right_root_level,
5472 advance_right != ADVANCE_ONLY_NEXT,
5473 &right_key);
Liu Bofb770ae2016-07-05 12:10:14 -07005474 if (ret == -1)
Alexander Block70698302012-06-05 21:07:48 +02005475 right_end_reached = ADVANCE;
Liu Bofb770ae2016-07-05 12:10:14 -07005476 else if (ret < 0)
5477 goto out;
Alexander Block70698302012-06-05 21:07:48 +02005478 advance_right = 0;
5479 }
5480
5481 if (left_end_reached && right_end_reached) {
5482 ret = 0;
5483 goto out;
5484 } else if (left_end_reached) {
5485 if (right_level == 0) {
Nikolay Borisovee8c4942017-08-21 12:43:45 +03005486 ret = changed_cb(left_path, right_path,
Alexander Block70698302012-06-05 21:07:48 +02005487 &right_key,
5488 BTRFS_COMPARE_TREE_DELETED,
5489 ctx);
5490 if (ret < 0)
5491 goto out;
5492 }
5493 advance_right = ADVANCE;
5494 continue;
5495 } else if (right_end_reached) {
5496 if (left_level == 0) {
Nikolay Borisovee8c4942017-08-21 12:43:45 +03005497 ret = changed_cb(left_path, right_path,
Alexander Block70698302012-06-05 21:07:48 +02005498 &left_key,
5499 BTRFS_COMPARE_TREE_NEW,
5500 ctx);
5501 if (ret < 0)
5502 goto out;
5503 }
5504 advance_left = ADVANCE;
5505 continue;
5506 }
5507
5508 if (left_level == 0 && right_level == 0) {
5509 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5510 if (cmp < 0) {
Nikolay Borisovee8c4942017-08-21 12:43:45 +03005511 ret = changed_cb(left_path, right_path,
Alexander Block70698302012-06-05 21:07:48 +02005512 &left_key,
5513 BTRFS_COMPARE_TREE_NEW,
5514 ctx);
5515 if (ret < 0)
5516 goto out;
5517 advance_left = ADVANCE;
5518 } else if (cmp > 0) {
Nikolay Borisovee8c4942017-08-21 12:43:45 +03005519 ret = changed_cb(left_path, right_path,
Alexander Block70698302012-06-05 21:07:48 +02005520 &right_key,
5521 BTRFS_COMPARE_TREE_DELETED,
5522 ctx);
5523 if (ret < 0)
5524 goto out;
5525 advance_right = ADVANCE;
5526 } else {
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005527 enum btrfs_compare_tree_result result;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005528
Chris Mason74dd17f2012-08-07 16:25:13 -04005529 WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04005530 ret = tree_compare_item(left_path, right_path,
5531 tmp_buf);
Josef Bacikba5e8f22013-08-16 16:52:55 -04005532 if (ret)
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005533 result = BTRFS_COMPARE_TREE_CHANGED;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005534 else
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005535 result = BTRFS_COMPARE_TREE_SAME;
Nikolay Borisovee8c4942017-08-21 12:43:45 +03005536 ret = changed_cb(left_path, right_path,
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005537 &left_key, result, ctx);
Josef Bacikba5e8f22013-08-16 16:52:55 -04005538 if (ret < 0)
5539 goto out;
Alexander Block70698302012-06-05 21:07:48 +02005540 advance_left = ADVANCE;
5541 advance_right = ADVANCE;
5542 }
5543 } else if (left_level == right_level) {
5544 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5545 if (cmp < 0) {
5546 advance_left = ADVANCE;
5547 } else if (cmp > 0) {
5548 advance_right = ADVANCE;
5549 } else {
5550 left_blockptr = btrfs_node_blockptr(
5551 left_path->nodes[left_level],
5552 left_path->slots[left_level]);
5553 right_blockptr = btrfs_node_blockptr(
5554 right_path->nodes[right_level],
5555 right_path->slots[right_level]);
Filipe Manana6baa4292014-02-20 21:15:25 +00005556 left_gen = btrfs_node_ptr_generation(
5557 left_path->nodes[left_level],
5558 left_path->slots[left_level]);
5559 right_gen = btrfs_node_ptr_generation(
5560 right_path->nodes[right_level],
5561 right_path->slots[right_level]);
5562 if (left_blockptr == right_blockptr &&
5563 left_gen == right_gen) {
Alexander Block70698302012-06-05 21:07:48 +02005564 /*
5565 * As we're on a shared block, don't
5566 * allow to go deeper.
5567 */
5568 advance_left = ADVANCE_ONLY_NEXT;
5569 advance_right = ADVANCE_ONLY_NEXT;
5570 } else {
5571 advance_left = ADVANCE;
5572 advance_right = ADVANCE;
5573 }
5574 }
5575 } else if (left_level < right_level) {
5576 advance_right = ADVANCE;
5577 } else {
5578 advance_left = ADVANCE;
5579 }
5580 }
5581
5582out:
5583 btrfs_free_path(left_path);
5584 btrfs_free_path(right_path);
David Sterba8f282f72016-03-30 16:01:12 +02005585 kvfree(tmp_buf);
Alexander Block70698302012-06-05 21:07:48 +02005586 return ret;
5587}
5588
Chris Mason3f157a22008-06-25 16:01:31 -04005589/*
5590 * this is similar to btrfs_next_leaf, but does not try to preserve
5591 * and fixup the path. It looks for and returns the next key in the
Eric Sandeende78b512013-01-31 18:21:12 +00005592 * tree based on the current path and the min_trans parameters.
Chris Mason3f157a22008-06-25 16:01:31 -04005593 *
5594 * 0 is returned if another key is found, < 0 if there are any errors
5595 * and 1 is returned if there are no higher keys in the tree
5596 *
5597 * path->keep_locks should be set to 1 on the search made before
5598 * calling this function.
5599 */
Chris Masone7a84562008-06-25 16:01:31 -04005600int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path,
Eric Sandeende78b512013-01-31 18:21:12 +00005601 struct btrfs_key *key, int level, u64 min_trans)
Chris Masone7a84562008-06-25 16:01:31 -04005602{
Chris Masone7a84562008-06-25 16:01:31 -04005603 int slot;
5604 struct extent_buffer *c;
5605
Chris Mason934d3752008-12-08 16:43:10 -05005606 WARN_ON(!path->keep_locks);
Chris Masond3977122009-01-05 21:25:51 -05005607 while (level < BTRFS_MAX_LEVEL) {
Chris Masone7a84562008-06-25 16:01:31 -04005608 if (!path->nodes[level])
5609 return 1;
5610
5611 slot = path->slots[level] + 1;
5612 c = path->nodes[level];
Chris Mason3f157a22008-06-25 16:01:31 -04005613next:
Chris Masone7a84562008-06-25 16:01:31 -04005614 if (slot >= btrfs_header_nritems(c)) {
Yan Zheng33c66f42009-07-22 09:59:00 -04005615 int ret;
5616 int orig_lowest;
5617 struct btrfs_key cur_key;
5618 if (level + 1 >= BTRFS_MAX_LEVEL ||
5619 !path->nodes[level + 1])
Chris Masone7a84562008-06-25 16:01:31 -04005620 return 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04005621
5622 if (path->locks[level + 1]) {
5623 level++;
5624 continue;
5625 }
5626
5627 slot = btrfs_header_nritems(c) - 1;
5628 if (level == 0)
5629 btrfs_item_key_to_cpu(c, &cur_key, slot);
5630 else
5631 btrfs_node_key_to_cpu(c, &cur_key, slot);
5632
5633 orig_lowest = path->lowest_level;
David Sterbab3b4aa72011-04-21 01:20:15 +02005634 btrfs_release_path(path);
Yan Zheng33c66f42009-07-22 09:59:00 -04005635 path->lowest_level = level;
5636 ret = btrfs_search_slot(NULL, root, &cur_key, path,
5637 0, 0);
5638 path->lowest_level = orig_lowest;
5639 if (ret < 0)
5640 return ret;
5641
5642 c = path->nodes[level];
5643 slot = path->slots[level];
5644 if (ret == 0)
5645 slot++;
5646 goto next;
Chris Masone7a84562008-06-25 16:01:31 -04005647 }
Yan Zheng33c66f42009-07-22 09:59:00 -04005648
Chris Masone7a84562008-06-25 16:01:31 -04005649 if (level == 0)
5650 btrfs_item_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005651 else {
Chris Mason3f157a22008-06-25 16:01:31 -04005652 u64 gen = btrfs_node_ptr_generation(c, slot);
5653
Chris Mason3f157a22008-06-25 16:01:31 -04005654 if (gen < min_trans) {
5655 slot++;
5656 goto next;
5657 }
Chris Masone7a84562008-06-25 16:01:31 -04005658 btrfs_node_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005659 }
Chris Masone7a84562008-06-25 16:01:31 -04005660 return 0;
5661 }
5662 return 1;
5663}
5664
Chris Mason7bb86312007-12-11 09:25:06 -05005665/*
Chris Mason925baed2008-06-25 16:01:30 -04005666 * search the tree again to find a leaf with greater keys
Chris Mason0f70abe2007-02-28 16:46:22 -05005667 * returns 0 if it found something or 1 if there are no greater leaves.
5668 * returns < 0 on io errors.
Chris Mason97571fd2007-02-24 13:39:08 -05005669 */
Chris Mason234b63a2007-03-13 10:46:10 -04005670int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Masond97e63b2007-02-20 16:40:44 -05005671{
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005672 return btrfs_next_old_leaf(root, path, 0);
5673}
5674
5675int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,
5676 u64 time_seq)
5677{
Chris Masond97e63b2007-02-20 16:40:44 -05005678 int slot;
Chris Mason8e73f272009-04-03 10:14:18 -04005679 int level;
Chris Mason5f39d392007-10-15 16:14:19 -04005680 struct extent_buffer *c;
Chris Mason8e73f272009-04-03 10:14:18 -04005681 struct extent_buffer *next;
Chris Mason925baed2008-06-25 16:01:30 -04005682 struct btrfs_key key;
5683 u32 nritems;
5684 int ret;
Chris Mason8e73f272009-04-03 10:14:18 -04005685 int old_spinning = path->leave_spinning;
Chris Masonbd681512011-07-16 15:23:14 -04005686 int next_rw_lock = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005687
5688 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Masond3977122009-01-05 21:25:51 -05005689 if (nritems == 0)
Chris Mason925baed2008-06-25 16:01:30 -04005690 return 1;
Chris Mason925baed2008-06-25 16:01:30 -04005691
Chris Mason8e73f272009-04-03 10:14:18 -04005692 btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1);
5693again:
5694 level = 1;
5695 next = NULL;
Chris Masonbd681512011-07-16 15:23:14 -04005696 next_rw_lock = 0;
David Sterbab3b4aa72011-04-21 01:20:15 +02005697 btrfs_release_path(path);
Chris Mason8e73f272009-04-03 10:14:18 -04005698
Chris Masona2135012008-06-25 16:01:30 -04005699 path->keep_locks = 1;
Chris Mason31533fb2011-07-26 16:01:59 -04005700 path->leave_spinning = 1;
Chris Mason8e73f272009-04-03 10:14:18 -04005701
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005702 if (time_seq)
5703 ret = btrfs_search_old_slot(root, &key, path, time_seq);
5704 else
5705 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
Chris Mason925baed2008-06-25 16:01:30 -04005706 path->keep_locks = 0;
5707
5708 if (ret < 0)
5709 return ret;
5710
Chris Masona2135012008-06-25 16:01:30 -04005711 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Mason168fd7d2008-06-25 16:01:30 -04005712 /*
5713 * by releasing the path above we dropped all our locks. A balance
5714 * could have added more items next to the key that used to be
5715 * at the very end of the block. So, check again here and
5716 * advance the path if there are now more items available.
5717 */
Chris Masona2135012008-06-25 16:01:30 -04005718 if (nritems > 0 && path->slots[0] < nritems - 1) {
Yan Zhenge457afe2009-07-22 09:59:00 -04005719 if (ret == 0)
5720 path->slots[0]++;
Chris Mason8e73f272009-04-03 10:14:18 -04005721 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005722 goto done;
5723 }
Liu Bo0b43e042014-06-09 11:04:49 +08005724 /*
5725 * So the above check misses one case:
5726 * - after releasing the path above, someone has removed the item that
5727 * used to be at the very end of the block, and balance between leafs
5728 * gets another one with bigger key.offset to replace it.
5729 *
5730 * This one should be returned as well, or we can get leaf corruption
5731 * later(esp. in __btrfs_drop_extents()).
5732 *
5733 * And a bit more explanation about this check,
5734 * with ret > 0, the key isn't found, the path points to the slot
5735 * where it should be inserted, so the path->slots[0] item must be the
5736 * bigger one.
5737 */
5738 if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) {
5739 ret = 0;
5740 goto done;
5741 }
Chris Masond97e63b2007-02-20 16:40:44 -05005742
Chris Masond3977122009-01-05 21:25:51 -05005743 while (level < BTRFS_MAX_LEVEL) {
Chris Mason8e73f272009-04-03 10:14:18 -04005744 if (!path->nodes[level]) {
5745 ret = 1;
5746 goto done;
5747 }
Chris Mason5f39d392007-10-15 16:14:19 -04005748
Chris Masond97e63b2007-02-20 16:40:44 -05005749 slot = path->slots[level] + 1;
5750 c = path->nodes[level];
Chris Mason5f39d392007-10-15 16:14:19 -04005751 if (slot >= btrfs_header_nritems(c)) {
Chris Masond97e63b2007-02-20 16:40:44 -05005752 level++;
Chris Mason8e73f272009-04-03 10:14:18 -04005753 if (level == BTRFS_MAX_LEVEL) {
5754 ret = 1;
5755 goto done;
5756 }
Chris Masond97e63b2007-02-20 16:40:44 -05005757 continue;
5758 }
Chris Mason5f39d392007-10-15 16:14:19 -04005759
Chris Mason925baed2008-06-25 16:01:30 -04005760 if (next) {
Chris Masonbd681512011-07-16 15:23:14 -04005761 btrfs_tree_unlock_rw(next, next_rw_lock);
Chris Mason5f39d392007-10-15 16:14:19 -04005762 free_extent_buffer(next);
Chris Mason925baed2008-06-25 16:01:30 -04005763 }
Chris Mason5f39d392007-10-15 16:14:19 -04005764
Chris Mason8e73f272009-04-03 10:14:18 -04005765 next = c;
Chris Masonbd681512011-07-16 15:23:14 -04005766 next_rw_lock = path->locks[level];
Liu Bod07b8522017-01-30 12:23:42 -08005767 ret = read_block_for_search(root, path, &next, level,
David Sterbacda79c52017-02-10 18:44:32 +01005768 slot, &key);
Chris Mason8e73f272009-04-03 10:14:18 -04005769 if (ret == -EAGAIN)
5770 goto again;
Chris Mason5f39d392007-10-15 16:14:19 -04005771
Chris Mason76a05b32009-05-14 13:24:30 -04005772 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005773 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005774 goto done;
5775 }
5776
Chris Mason5cd57b22008-06-25 16:01:30 -04005777 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005778 ret = btrfs_try_tree_read_lock(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005779 if (!ret && time_seq) {
5780 /*
5781 * If we don't get the lock, we may be racing
5782 * with push_leaf_left, holding that lock while
5783 * itself waiting for the leaf we've currently
5784 * locked. To solve this situation, we give up
5785 * on our lock and cycle.
5786 */
Jan Schmidtcf538832012-07-04 15:42:48 +02005787 free_extent_buffer(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005788 btrfs_release_path(path);
5789 cond_resched();
5790 goto again;
5791 }
Chris Mason8e73f272009-04-03 10:14:18 -04005792 if (!ret) {
5793 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005794 btrfs_tree_read_lock(next);
Chris Mason8e73f272009-04-03 10:14:18 -04005795 }
Chris Mason31533fb2011-07-26 16:01:59 -04005796 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005797 }
Chris Masond97e63b2007-02-20 16:40:44 -05005798 break;
5799 }
5800 path->slots[level] = slot;
Chris Masond3977122009-01-05 21:25:51 -05005801 while (1) {
Chris Masond97e63b2007-02-20 16:40:44 -05005802 level--;
5803 c = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04005804 if (path->locks[level])
Chris Masonbd681512011-07-16 15:23:14 -04005805 btrfs_tree_unlock_rw(c, path->locks[level]);
Chris Mason8e73f272009-04-03 10:14:18 -04005806
Chris Mason5f39d392007-10-15 16:14:19 -04005807 free_extent_buffer(c);
Chris Masond97e63b2007-02-20 16:40:44 -05005808 path->nodes[level] = next;
5809 path->slots[level] = 0;
Chris Masona74a4b92008-06-25 16:01:31 -04005810 if (!path->skip_locking)
Chris Masonbd681512011-07-16 15:23:14 -04005811 path->locks[level] = next_rw_lock;
Chris Masond97e63b2007-02-20 16:40:44 -05005812 if (!level)
5813 break;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005814
Liu Bod07b8522017-01-30 12:23:42 -08005815 ret = read_block_for_search(root, path, &next, level,
David Sterbacda79c52017-02-10 18:44:32 +01005816 0, &key);
Chris Mason8e73f272009-04-03 10:14:18 -04005817 if (ret == -EAGAIN)
5818 goto again;
5819
Chris Mason76a05b32009-05-14 13:24:30 -04005820 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005821 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005822 goto done;
5823 }
5824
Chris Mason5cd57b22008-06-25 16:01:30 -04005825 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005826 ret = btrfs_try_tree_read_lock(next);
Chris Mason8e73f272009-04-03 10:14:18 -04005827 if (!ret) {
5828 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005829 btrfs_tree_read_lock(next);
Chris Mason8e73f272009-04-03 10:14:18 -04005830 }
Chris Mason31533fb2011-07-26 16:01:59 -04005831 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005832 }
Chris Masond97e63b2007-02-20 16:40:44 -05005833 }
Chris Mason8e73f272009-04-03 10:14:18 -04005834 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005835done:
Chris Masonf7c79f32012-03-19 15:54:38 -04005836 unlock_up(path, 0, 1, 0, NULL);
Chris Mason8e73f272009-04-03 10:14:18 -04005837 path->leave_spinning = old_spinning;
5838 if (!old_spinning)
5839 btrfs_set_path_blocking(path);
5840
5841 return ret;
Chris Masond97e63b2007-02-20 16:40:44 -05005842}
Chris Mason0b86a832008-03-24 15:01:56 -04005843
Chris Mason3f157a22008-06-25 16:01:31 -04005844/*
5845 * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps
5846 * searching until it gets past min_objectid or finds an item of 'type'
5847 *
5848 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5849 */
Chris Mason0b86a832008-03-24 15:01:56 -04005850int btrfs_previous_item(struct btrfs_root *root,
5851 struct btrfs_path *path, u64 min_objectid,
5852 int type)
5853{
5854 struct btrfs_key found_key;
5855 struct extent_buffer *leaf;
Chris Masone02119d2008-09-05 16:13:11 -04005856 u32 nritems;
Chris Mason0b86a832008-03-24 15:01:56 -04005857 int ret;
5858
Chris Masond3977122009-01-05 21:25:51 -05005859 while (1) {
Chris Mason0b86a832008-03-24 15:01:56 -04005860 if (path->slots[0] == 0) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05005861 btrfs_set_path_blocking(path);
Chris Mason0b86a832008-03-24 15:01:56 -04005862 ret = btrfs_prev_leaf(root, path);
5863 if (ret != 0)
5864 return ret;
5865 } else {
5866 path->slots[0]--;
5867 }
5868 leaf = path->nodes[0];
Chris Masone02119d2008-09-05 16:13:11 -04005869 nritems = btrfs_header_nritems(leaf);
5870 if (nritems == 0)
5871 return 1;
5872 if (path->slots[0] == nritems)
5873 path->slots[0]--;
5874
Chris Mason0b86a832008-03-24 15:01:56 -04005875 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
Chris Masone02119d2008-09-05 16:13:11 -04005876 if (found_key.objectid < min_objectid)
5877 break;
Yan Zheng0a4eefb2009-07-24 11:06:53 -04005878 if (found_key.type == type)
5879 return 0;
Chris Masone02119d2008-09-05 16:13:11 -04005880 if (found_key.objectid == min_objectid &&
5881 found_key.type < type)
5882 break;
Chris Mason0b86a832008-03-24 15:01:56 -04005883 }
5884 return 1;
5885}
Wang Shilongade2e0b2014-01-12 21:38:33 +08005886
5887/*
5888 * search in extent tree to find a previous Metadata/Data extent item with
5889 * min objecitd.
5890 *
5891 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5892 */
5893int btrfs_previous_extent_item(struct btrfs_root *root,
5894 struct btrfs_path *path, u64 min_objectid)
5895{
5896 struct btrfs_key found_key;
5897 struct extent_buffer *leaf;
5898 u32 nritems;
5899 int ret;
5900
5901 while (1) {
5902 if (path->slots[0] == 0) {
5903 btrfs_set_path_blocking(path);
5904 ret = btrfs_prev_leaf(root, path);
5905 if (ret != 0)
5906 return ret;
5907 } else {
5908 path->slots[0]--;
5909 }
5910 leaf = path->nodes[0];
5911 nritems = btrfs_header_nritems(leaf);
5912 if (nritems == 0)
5913 return 1;
5914 if (path->slots[0] == nritems)
5915 path->slots[0]--;
5916
5917 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5918 if (found_key.objectid < min_objectid)
5919 break;
5920 if (found_key.type == BTRFS_EXTENT_ITEM_KEY ||
5921 found_key.type == BTRFS_METADATA_ITEM_KEY)
5922 return 0;
5923 if (found_key.objectid == min_objectid &&
5924 found_key.type < BTRFS_EXTENT_ITEM_KEY)
5925 break;
5926 }
5927 return 1;
5928}