blob: 88deec01238626d6ae9b868da2aabb434dbb6690 [file] [log] [blame]
David Sterbac1d7c512018-04-03 19:23:33 +02001// SPDX-License-Identifier: GPL-2.0
Chris Mason6cbd5572007-06-12 09:07:21 -04002/*
3 * Copyright (C) 2007 Oracle. All rights reserved.
Chris Mason6cbd5572007-06-12 09:07:21 -04004 */
David Sterbac1d7c512018-04-03 19:23:33 +02005
Zach Brownec6b9102007-07-11 10:00:37 -04006#include <linux/sched.h>
Ingo Molnarf361bf42017-02-03 23:47:37 +01007#include <linux/sched/signal.h>
Chris Masonedbd8d42007-12-21 16:27:24 -05008#include <linux/pagemap.h>
Chris Masonec44a352008-04-28 15:29:52 -04009#include <linux/writeback.h>
David Woodhouse21af8042008-08-12 14:13:26 +010010#include <linux/blkdev.h>
Chris Masonb7a9f292009-02-04 09:23:45 -050011#include <linux/sort.h>
Chris Mason4184ea72009-03-10 12:39:20 -040012#include <linux/rcupdate.h>
Josef Bacik817d52f2009-07-13 21:29:25 -040013#include <linux/kthread.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090014#include <linux/slab.h>
David Sterbadff51cd2011-06-14 12:52:17 +020015#include <linux/ratelimit.h>
Josef Bacikb150a4f2013-06-19 15:00:04 -040016#include <linux/percpu_counter.h>
Josef Bacik69fe2d72017-10-19 14:15:57 -040017#include <linux/lockdep.h>
Nikolay Borisov9678c542018-01-08 11:45:05 +020018#include <linux/crc32c.h>
Miao Xie995946d2014-04-02 19:51:06 +080019#include "tree-log.h"
Chris Masonfec577f2007-02-26 10:40:21 -050020#include "disk-io.h"
21#include "print-tree.h"
Chris Mason0b86a832008-03-24 15:01:56 -040022#include "volumes.h"
David Woodhouse53b381b2013-01-29 18:40:14 -050023#include "raid56.h"
Chris Mason925baed2008-06-25 16:01:30 -040024#include "locking.h"
Chris Masonfa9c0d792009-04-03 09:47:43 -040025#include "free-space-cache.h"
Omar Sandoval1e144fb2015-09-29 20:50:37 -070026#include "free-space-tree.h"
Miao Xie3fed40c2012-09-13 04:51:36 -060027#include "math.h"
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040028#include "sysfs.h"
Josef Bacikfcebe452014-05-13 17:30:47 -070029#include "qgroup.h"
Josef Bacikfd708b82017-09-29 15:43:50 -040030#include "ref-verify.h"
Chris Masonfec577f2007-02-26 10:40:21 -050031
Arne Jansen709c0482011-09-12 12:22:57 +020032#undef SCRAMBLE_DELAYED_REFS
33
Miao Xie9e622d62012-01-26 15:01:12 -050034/*
35 * control flags for do_chunk_alloc's force field
Chris Mason0e4f8f82011-04-15 16:05:44 -040036 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
37 * if we really need one.
38 *
Chris Mason0e4f8f82011-04-15 16:05:44 -040039 * CHUNK_ALLOC_LIMITED means to only try and allocate one
40 * if we have very few chunks already allocated. This is
41 * used as part of the clustering code to help make sure
42 * we have a good pool of storage to cluster in, without
43 * filling the FS with empty chunks
44 *
Miao Xie9e622d62012-01-26 15:01:12 -050045 * CHUNK_ALLOC_FORCE means it must try to allocate one
46 *
Chris Mason0e4f8f82011-04-15 16:05:44 -040047 */
48enum {
49 CHUNK_ALLOC_NO_FORCE = 0,
Miao Xie9e622d62012-01-26 15:01:12 -050050 CHUNK_ALLOC_LIMITED = 1,
51 CHUNK_ALLOC_FORCE = 2,
Chris Mason0e4f8f82011-04-15 16:05:44 -040052};
53
Qu Wenruo9f9b8e82018-10-24 20:24:01 +080054/*
55 * Declare a helper function to detect underflow of various space info members
56 */
57#define DECLARE_SPACE_INFO_UPDATE(name) \
58static inline void update_##name(struct btrfs_space_info *sinfo, \
59 s64 bytes) \
60{ \
61 if (bytes < 0 && sinfo->name < -bytes) { \
62 WARN_ON(1); \
63 sinfo->name = 0; \
64 return; \
65 } \
66 sinfo->name += bytes; \
67}
68
69DECLARE_SPACE_INFO_UPDATE(bytes_may_use);
Lu Fengqie2907c12018-10-24 20:24:02 +080070DECLARE_SPACE_INFO_UPDATE(bytes_pinned);
Qu Wenruo9f9b8e82018-10-24 20:24:01 +080071
Yan Zheng5d4f98a2009-06-10 10:45:14 -040072static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
Nikolay Borisove72cb922018-06-20 15:48:57 +030073 struct btrfs_delayed_ref_node *node, u64 parent,
74 u64 root_objectid, u64 owner_objectid,
75 u64 owner_offset, int refs_to_drop,
76 struct btrfs_delayed_extent_op *extra_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -040077static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
78 struct extent_buffer *leaf,
79 struct btrfs_extent_item *ei);
80static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -040081 u64 parent, u64 root_objectid,
82 u64 flags, u64 owner, u64 offset,
83 struct btrfs_key *ins, int ref_mod);
84static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +030085 struct btrfs_delayed_ref_node *node,
Nikolay Borisov21ebfbe2018-05-21 12:27:22 +030086 struct btrfs_delayed_extent_op *extent_op);
Nikolay Borisov01458822018-06-20 15:49:05 +030087static int do_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags,
Josef Bacik698d0082012-09-12 14:08:47 -040088 int force);
Yan Zheng11833d62009-09-11 16:11:19 -040089static int find_next_key(struct btrfs_path *path, int level,
90 struct btrfs_key *key);
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -040091static void dump_space_info(struct btrfs_fs_info *fs_info,
92 struct btrfs_space_info *info, u64 bytes,
Josef Bacik9ed74f22009-09-11 16:12:44 -040093 int dump_block_groups);
Josef Bacik5d803662013-02-07 16:06:02 -050094static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
95 u64 num_bytes);
Josef Bacik957780e2016-05-17 13:30:55 -040096static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info,
97 struct btrfs_space_info *space_info,
98 u64 num_bytes);
99static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info,
100 struct btrfs_space_info *space_info,
101 u64 num_bytes);
Josef Bacik6a632092009-02-20 11:00:09 -0500102
Josef Bacik817d52f2009-07-13 21:29:25 -0400103static noinline int
104block_group_cache_done(struct btrfs_block_group_cache *cache)
105{
106 smp_mb();
Josef Bacik36cce922013-08-05 11:15:21 -0400107 return cache->cached == BTRFS_CACHE_FINISHED ||
108 cache->cached == BTRFS_CACHE_ERROR;
Josef Bacik817d52f2009-07-13 21:29:25 -0400109}
110
Josef Bacik0f9dd462008-09-23 13:14:11 -0400111static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
112{
113 return (cache->flags & bits) == bits;
114}
115
Filipe Manana758f2df2015-11-19 11:45:48 +0000116void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
Josef Bacik11dfe352009-11-13 20:12:59 +0000117{
118 atomic_inc(&cache->count);
119}
120
121void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
122{
Yan, Zhengf0486c62010-05-16 10:46:25 -0400123 if (atomic_dec_and_test(&cache->count)) {
124 WARN_ON(cache->pinned > 0);
125 WARN_ON(cache->reserved > 0);
Qu Wenruo0966a7b2017-04-14 08:35:54 +0800126
127 /*
128 * If not empty, someone is still holding mutex of
129 * full_stripe_lock, which can only be released by caller.
130 * And it will definitely cause use-after-free when caller
131 * tries to release full stripe lock.
132 *
133 * No better way to resolve, but only to warn.
134 */
135 WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root));
Li Zefan34d52cb2011-03-29 13:46:06 +0800136 kfree(cache->free_space_ctl);
Josef Bacik11dfe352009-11-13 20:12:59 +0000137 kfree(cache);
Yan, Zhengf0486c62010-05-16 10:46:25 -0400138 }
Josef Bacik11dfe352009-11-13 20:12:59 +0000139}
140
Josef Bacik0f9dd462008-09-23 13:14:11 -0400141/*
142 * this adds the block group to the fs_info rb tree for the block group
143 * cache
144 */
Christoph Hellwigb2950862008-12-02 09:54:17 -0500145static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
Josef Bacik0f9dd462008-09-23 13:14:11 -0400146 struct btrfs_block_group_cache *block_group)
147{
148 struct rb_node **p;
149 struct rb_node *parent = NULL;
150 struct btrfs_block_group_cache *cache;
151
152 spin_lock(&info->block_group_cache_lock);
153 p = &info->block_group_cache_tree.rb_node;
154
155 while (*p) {
156 parent = *p;
157 cache = rb_entry(parent, struct btrfs_block_group_cache,
158 cache_node);
159 if (block_group->key.objectid < cache->key.objectid) {
160 p = &(*p)->rb_left;
161 } else if (block_group->key.objectid > cache->key.objectid) {
162 p = &(*p)->rb_right;
163 } else {
164 spin_unlock(&info->block_group_cache_lock);
165 return -EEXIST;
166 }
167 }
168
169 rb_link_node(&block_group->cache_node, parent, p);
170 rb_insert_color(&block_group->cache_node,
171 &info->block_group_cache_tree);
Liu Boa1897fd2012-12-27 09:01:23 +0000172
173 if (info->first_logical_byte > block_group->key.objectid)
174 info->first_logical_byte = block_group->key.objectid;
175
Josef Bacik0f9dd462008-09-23 13:14:11 -0400176 spin_unlock(&info->block_group_cache_lock);
177
178 return 0;
179}
180
181/*
182 * This will return the block group at or after bytenr if contains is 0, else
183 * it will return the block group that contains the bytenr
184 */
185static struct btrfs_block_group_cache *
186block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
187 int contains)
188{
189 struct btrfs_block_group_cache *cache, *ret = NULL;
190 struct rb_node *n;
191 u64 end, start;
192
193 spin_lock(&info->block_group_cache_lock);
194 n = info->block_group_cache_tree.rb_node;
195
196 while (n) {
197 cache = rb_entry(n, struct btrfs_block_group_cache,
198 cache_node);
199 end = cache->key.objectid + cache->key.offset - 1;
200 start = cache->key.objectid;
201
202 if (bytenr < start) {
203 if (!contains && (!ret || start < ret->key.objectid))
204 ret = cache;
205 n = n->rb_left;
206 } else if (bytenr > start) {
207 if (contains && bytenr <= end) {
208 ret = cache;
209 break;
210 }
211 n = n->rb_right;
212 } else {
213 ret = cache;
214 break;
215 }
216 }
Liu Boa1897fd2012-12-27 09:01:23 +0000217 if (ret) {
Josef Bacik11dfe352009-11-13 20:12:59 +0000218 btrfs_get_block_group(ret);
Liu Boa1897fd2012-12-27 09:01:23 +0000219 if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
220 info->first_logical_byte = ret->key.objectid;
221 }
Josef Bacik0f9dd462008-09-23 13:14:11 -0400222 spin_unlock(&info->block_group_cache_lock);
223
224 return ret;
225}
226
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400227static int add_excluded_extent(struct btrfs_fs_info *fs_info,
Yan Zheng11833d62009-09-11 16:11:19 -0400228 u64 start, u64 num_bytes)
Josef Bacik817d52f2009-07-13 21:29:25 -0400229{
Yan Zheng11833d62009-09-11 16:11:19 -0400230 u64 end = start + num_bytes - 1;
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400231 set_extent_bits(&fs_info->freed_extents[0],
David Sterbaceeb0ae2016-04-26 23:54:39 +0200232 start, end, EXTENT_UPTODATE);
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400233 set_extent_bits(&fs_info->freed_extents[1],
David Sterbaceeb0ae2016-04-26 23:54:39 +0200234 start, end, EXTENT_UPTODATE);
Yan Zheng11833d62009-09-11 16:11:19 -0400235 return 0;
Josef Bacik817d52f2009-07-13 21:29:25 -0400236}
237
Nikolay Borisov9e715da2018-06-20 15:49:08 +0300238static void free_excluded_extents(struct btrfs_block_group_cache *cache)
Josef Bacik817d52f2009-07-13 21:29:25 -0400239{
Nikolay Borisov9e715da2018-06-20 15:49:08 +0300240 struct btrfs_fs_info *fs_info = cache->fs_info;
Yan Zheng11833d62009-09-11 16:11:19 -0400241 u64 start, end;
242
243 start = cache->key.objectid;
244 end = start + cache->key.offset - 1;
245
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400246 clear_extent_bits(&fs_info->freed_extents[0],
David Sterba91166212016-04-26 23:54:39 +0200247 start, end, EXTENT_UPTODATE);
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400248 clear_extent_bits(&fs_info->freed_extents[1],
David Sterba91166212016-04-26 23:54:39 +0200249 start, end, EXTENT_UPTODATE);
Yan Zheng11833d62009-09-11 16:11:19 -0400250}
251
Nikolay Borisov3c4da652018-06-20 15:49:09 +0300252static int exclude_super_stripes(struct btrfs_block_group_cache *cache)
Yan Zheng11833d62009-09-11 16:11:19 -0400253{
Nikolay Borisov3c4da652018-06-20 15:49:09 +0300254 struct btrfs_fs_info *fs_info = cache->fs_info;
Josef Bacik817d52f2009-07-13 21:29:25 -0400255 u64 bytenr;
256 u64 *logical;
257 int stripe_len;
258 int i, nr, ret;
259
Yan, Zheng06b23312009-11-26 09:31:11 +0000260 if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
261 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
262 cache->bytes_super += stripe_len;
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400263 ret = add_excluded_extent(fs_info, cache->key.objectid,
Yan, Zheng06b23312009-11-26 09:31:11 +0000264 stripe_len);
Josef Bacik835d9742013-03-19 12:13:25 -0400265 if (ret)
266 return ret;
Yan, Zheng06b23312009-11-26 09:31:11 +0000267 }
268
Josef Bacik817d52f2009-07-13 21:29:25 -0400269 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
270 bytenr = btrfs_sb_offset(i);
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400271 ret = btrfs_rmap_block(fs_info, cache->key.objectid,
Nikolay Borisov63a9c7b2018-05-04 10:53:05 +0300272 bytenr, &logical, &nr, &stripe_len);
Josef Bacik835d9742013-03-19 12:13:25 -0400273 if (ret)
274 return ret;
Yan Zheng11833d62009-09-11 16:11:19 -0400275
Josef Bacik817d52f2009-07-13 21:29:25 -0400276 while (nr--) {
Josef Bacik51bf5f02013-04-23 12:55:21 -0400277 u64 start, len;
278
279 if (logical[nr] > cache->key.objectid +
280 cache->key.offset)
281 continue;
282
283 if (logical[nr] + stripe_len <= cache->key.objectid)
284 continue;
285
286 start = logical[nr];
287 if (start < cache->key.objectid) {
288 start = cache->key.objectid;
289 len = (logical[nr] + stripe_len) - start;
290 } else {
291 len = min_t(u64, stripe_len,
292 cache->key.objectid +
293 cache->key.offset - start);
294 }
295
296 cache->bytes_super += len;
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400297 ret = add_excluded_extent(fs_info, start, len);
Josef Bacik835d9742013-03-19 12:13:25 -0400298 if (ret) {
299 kfree(logical);
300 return ret;
301 }
Josef Bacik817d52f2009-07-13 21:29:25 -0400302 }
Yan Zheng11833d62009-09-11 16:11:19 -0400303
Josef Bacik817d52f2009-07-13 21:29:25 -0400304 kfree(logical);
305 }
Josef Bacik817d52f2009-07-13 21:29:25 -0400306 return 0;
307}
308
Yan Zheng11833d62009-09-11 16:11:19 -0400309static struct btrfs_caching_control *
310get_caching_control(struct btrfs_block_group_cache *cache)
311{
312 struct btrfs_caching_control *ctl;
313
314 spin_lock(&cache->lock);
Josef Bacikdde5abe2010-09-16 16:17:03 -0400315 if (!cache->caching_ctl) {
316 spin_unlock(&cache->lock);
317 return NULL;
318 }
319
Yan Zheng11833d62009-09-11 16:11:19 -0400320 ctl = cache->caching_ctl;
Elena Reshetova1e4f4712017-03-03 10:55:14 +0200321 refcount_inc(&ctl->count);
Yan Zheng11833d62009-09-11 16:11:19 -0400322 spin_unlock(&cache->lock);
323 return ctl;
324}
325
326static void put_caching_control(struct btrfs_caching_control *ctl)
327{
Elena Reshetova1e4f4712017-03-03 10:55:14 +0200328 if (refcount_dec_and_test(&ctl->count))
Yan Zheng11833d62009-09-11 16:11:19 -0400329 kfree(ctl);
330}
331
Josef Bacikd0bd4562015-09-23 14:54:14 -0400332#ifdef CONFIG_BTRFS_DEBUG
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400333static void fragment_free_space(struct btrfs_block_group_cache *block_group)
Josef Bacikd0bd4562015-09-23 14:54:14 -0400334{
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400335 struct btrfs_fs_info *fs_info = block_group->fs_info;
Josef Bacikd0bd4562015-09-23 14:54:14 -0400336 u64 start = block_group->key.objectid;
337 u64 len = block_group->key.offset;
338 u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ?
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400339 fs_info->nodesize : fs_info->sectorsize;
Josef Bacikd0bd4562015-09-23 14:54:14 -0400340 u64 step = chunk << 1;
341
342 while (len > chunk) {
343 btrfs_remove_free_space(block_group, start, chunk);
344 start += step;
345 if (len < step)
346 len = 0;
347 else
348 len -= step;
349 }
350}
351#endif
352
Josef Bacik0f9dd462008-09-23 13:14:11 -0400353/*
354 * this is only called by cache_block_group, since we could have freed extents
355 * we need to check the pinned_extents for any extents that can't be used yet
356 * since their free space will be released as soon as the transaction commits.
357 */
Omar Sandovala5ed9182015-09-29 20:50:35 -0700358u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
Nikolay Borisov4457c1c2018-05-10 15:44:45 +0300359 u64 start, u64 end)
Josef Bacik0f9dd462008-09-23 13:14:11 -0400360{
Nikolay Borisov4457c1c2018-05-10 15:44:45 +0300361 struct btrfs_fs_info *info = block_group->fs_info;
Josef Bacik817d52f2009-07-13 21:29:25 -0400362 u64 extent_start, extent_end, size, total_added = 0;
Josef Bacik0f9dd462008-09-23 13:14:11 -0400363 int ret;
364
365 while (start < end) {
Yan Zheng11833d62009-09-11 16:11:19 -0400366 ret = find_first_extent_bit(info->pinned_extents, start,
Josef Bacik0f9dd462008-09-23 13:14:11 -0400367 &extent_start, &extent_end,
Josef Bacike6138872012-09-27 17:07:30 -0400368 EXTENT_DIRTY | EXTENT_UPTODATE,
369 NULL);
Josef Bacik0f9dd462008-09-23 13:14:11 -0400370 if (ret)
371 break;
372
Yan, Zheng06b23312009-11-26 09:31:11 +0000373 if (extent_start <= start) {
Josef Bacik0f9dd462008-09-23 13:14:11 -0400374 start = extent_end + 1;
375 } else if (extent_start > start && extent_start < end) {
376 size = extent_start - start;
Josef Bacik817d52f2009-07-13 21:29:25 -0400377 total_added += size;
Josef Bacikea6a4782008-11-20 12:16:16 -0500378 ret = btrfs_add_free_space(block_group, start,
379 size);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100380 BUG_ON(ret); /* -ENOMEM or logic error */
Josef Bacik0f9dd462008-09-23 13:14:11 -0400381 start = extent_end + 1;
382 } else {
383 break;
384 }
385 }
386
387 if (start < end) {
388 size = end - start;
Josef Bacik817d52f2009-07-13 21:29:25 -0400389 total_added += size;
Josef Bacikea6a4782008-11-20 12:16:16 -0500390 ret = btrfs_add_free_space(block_group, start, size);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100391 BUG_ON(ret); /* -ENOMEM or logic error */
Josef Bacik0f9dd462008-09-23 13:14:11 -0400392 }
393
Josef Bacik817d52f2009-07-13 21:29:25 -0400394 return total_added;
Josef Bacik0f9dd462008-09-23 13:14:11 -0400395}
396
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700397static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
Chris Masone37c9e62007-05-09 20:13:14 -0400398{
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400399 struct btrfs_block_group_cache *block_group = caching_ctl->block_group;
400 struct btrfs_fs_info *fs_info = block_group->fs_info;
401 struct btrfs_root *extent_root = fs_info->extent_root;
Chris Masone37c9e62007-05-09 20:13:14 -0400402 struct btrfs_path *path;
Chris Mason5f39d392007-10-15 16:14:19 -0400403 struct extent_buffer *leaf;
Yan Zheng11833d62009-09-11 16:11:19 -0400404 struct btrfs_key key;
Josef Bacik817d52f2009-07-13 21:29:25 -0400405 u64 total_found = 0;
Yan Zheng11833d62009-09-11 16:11:19 -0400406 u64 last = 0;
407 u32 nritems;
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700408 int ret;
Josef Bacikd0bd4562015-09-23 14:54:14 -0400409 bool wakeup = true;
Chris Masonf510cfe2007-10-15 16:14:48 -0400410
Chris Masone37c9e62007-05-09 20:13:14 -0400411 path = btrfs_alloc_path();
412 if (!path)
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700413 return -ENOMEM;
Yan7d7d6062007-09-14 16:15:28 -0400414
Josef Bacik817d52f2009-07-13 21:29:25 -0400415 last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
Yan Zheng11833d62009-09-11 16:11:19 -0400416
Josef Bacikd0bd4562015-09-23 14:54:14 -0400417#ifdef CONFIG_BTRFS_DEBUG
418 /*
419 * If we're fragmenting we don't want to make anybody think we can
420 * allocate from this block group until we've had a chance to fragment
421 * the free space.
422 */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400423 if (btrfs_should_fragment_free_space(block_group))
Josef Bacikd0bd4562015-09-23 14:54:14 -0400424 wakeup = false;
425#endif
Chris Mason5cd57b22008-06-25 16:01:30 -0400426 /*
Josef Bacik817d52f2009-07-13 21:29:25 -0400427 * We don't want to deadlock with somebody trying to allocate a new
428 * extent for the extent root while also trying to search the extent
429 * root to add free space. So we skip locking and search the commit
430 * root, since its read-only
Chris Mason5cd57b22008-06-25 16:01:30 -0400431 */
432 path->skip_locking = 1;
Josef Bacik817d52f2009-07-13 21:29:25 -0400433 path->search_commit_root = 1;
David Sterbae4058b52015-11-27 16:31:35 +0100434 path->reada = READA_FORWARD;
Josef Bacik817d52f2009-07-13 21:29:25 -0400435
Yan Zhenge4404d62008-12-12 10:03:26 -0500436 key.objectid = last;
Chris Masone37c9e62007-05-09 20:13:14 -0400437 key.offset = 0;
Yan Zheng11833d62009-09-11 16:11:19 -0400438 key.type = BTRFS_EXTENT_ITEM_KEY;
Chris Mason013f1b12009-07-31 14:57:55 -0400439
Liu Bo52ee28d2013-07-11 17:51:15 +0800440next:
Yan Zheng11833d62009-09-11 16:11:19 -0400441 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
Chris Masone37c9e62007-05-09 20:13:14 -0400442 if (ret < 0)
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700443 goto out;
Yan Zhenga512bbf2008-12-08 16:46:26 -0500444
Yan Zheng11833d62009-09-11 16:11:19 -0400445 leaf = path->nodes[0];
446 nritems = btrfs_header_nritems(leaf);
447
Chris Masond3977122009-01-05 21:25:51 -0500448 while (1) {
David Sterba7841cb22011-05-31 18:07:27 +0200449 if (btrfs_fs_closing(fs_info) > 1) {
Yan Zhengf25784b2009-07-28 08:41:57 -0400450 last = (u64)-1;
Josef Bacik817d52f2009-07-13 21:29:25 -0400451 break;
Yan Zhengf25784b2009-07-28 08:41:57 -0400452 }
Josef Bacik817d52f2009-07-13 21:29:25 -0400453
Yan Zheng11833d62009-09-11 16:11:19 -0400454 if (path->slots[0] < nritems) {
455 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
456 } else {
457 ret = find_next_key(path, 0, &key);
458 if (ret)
Chris Masone37c9e62007-05-09 20:13:14 -0400459 break;
Josef Bacik817d52f2009-07-13 21:29:25 -0400460
Josef Bacikc9ea7b22013-09-19 10:02:11 -0400461 if (need_resched() ||
Josef Bacik9e351cc2014-03-13 15:42:13 -0400462 rwsem_is_contended(&fs_info->commit_root_sem)) {
Josef Bacikd0bd4562015-09-23 14:54:14 -0400463 if (wakeup)
464 caching_ctl->progress = last;
Chris Masonff5714c2011-05-28 07:00:39 -0400465 btrfs_release_path(path);
Josef Bacik9e351cc2014-03-13 15:42:13 -0400466 up_read(&fs_info->commit_root_sem);
Josef Bacik589d8ad2011-05-11 17:30:53 -0400467 mutex_unlock(&caching_ctl->mutex);
Yan Zheng11833d62009-09-11 16:11:19 -0400468 cond_resched();
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700469 mutex_lock(&caching_ctl->mutex);
470 down_read(&fs_info->commit_root_sem);
471 goto next;
Josef Bacik589d8ad2011-05-11 17:30:53 -0400472 }
Josef Bacik0a3896d2013-04-19 14:37:26 -0400473
474 ret = btrfs_next_leaf(extent_root, path);
475 if (ret < 0)
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700476 goto out;
Josef Bacik0a3896d2013-04-19 14:37:26 -0400477 if (ret)
478 break;
Josef Bacik589d8ad2011-05-11 17:30:53 -0400479 leaf = path->nodes[0];
480 nritems = btrfs_header_nritems(leaf);
481 continue;
Yan Zheng11833d62009-09-11 16:11:19 -0400482 }
Josef Bacik817d52f2009-07-13 21:29:25 -0400483
Liu Bo52ee28d2013-07-11 17:51:15 +0800484 if (key.objectid < last) {
485 key.objectid = last;
486 key.offset = 0;
487 key.type = BTRFS_EXTENT_ITEM_KEY;
488
Josef Bacikd0bd4562015-09-23 14:54:14 -0400489 if (wakeup)
490 caching_ctl->progress = last;
Liu Bo52ee28d2013-07-11 17:51:15 +0800491 btrfs_release_path(path);
492 goto next;
493 }
494
Yan Zheng11833d62009-09-11 16:11:19 -0400495 if (key.objectid < block_group->key.objectid) {
496 path->slots[0]++;
Josef Bacik817d52f2009-07-13 21:29:25 -0400497 continue;
Chris Masone37c9e62007-05-09 20:13:14 -0400498 }
Josef Bacik0f9dd462008-09-23 13:14:11 -0400499
Chris Masone37c9e62007-05-09 20:13:14 -0400500 if (key.objectid >= block_group->key.objectid +
Josef Bacik0f9dd462008-09-23 13:14:11 -0400501 block_group->key.offset)
Yan7d7d6062007-09-14 16:15:28 -0400502 break;
Yan7d7d6062007-09-14 16:15:28 -0400503
Josef Bacik3173a182013-03-07 14:22:04 -0500504 if (key.type == BTRFS_EXTENT_ITEM_KEY ||
505 key.type == BTRFS_METADATA_ITEM_KEY) {
Nikolay Borisov4457c1c2018-05-10 15:44:45 +0300506 total_found += add_new_free_space(block_group, last,
Josef Bacik817d52f2009-07-13 21:29:25 -0400507 key.objectid);
Josef Bacik3173a182013-03-07 14:22:04 -0500508 if (key.type == BTRFS_METADATA_ITEM_KEY)
509 last = key.objectid +
Jeff Mahoneyda170662016-06-15 09:22:56 -0400510 fs_info->nodesize;
Josef Bacik3173a182013-03-07 14:22:04 -0500511 else
512 last = key.objectid + key.offset;
Josef Bacik817d52f2009-07-13 21:29:25 -0400513
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700514 if (total_found > CACHING_CTL_WAKE_UP) {
Yan Zheng11833d62009-09-11 16:11:19 -0400515 total_found = 0;
Josef Bacikd0bd4562015-09-23 14:54:14 -0400516 if (wakeup)
517 wake_up(&caching_ctl->wait);
Yan Zheng11833d62009-09-11 16:11:19 -0400518 }
Josef Bacik817d52f2009-07-13 21:29:25 -0400519 }
Chris Masone37c9e62007-05-09 20:13:14 -0400520 path->slots[0]++;
521 }
Josef Bacikef8bbdf2008-09-23 13:14:11 -0400522 ret = 0;
Josef Bacik817d52f2009-07-13 21:29:25 -0400523
Nikolay Borisov4457c1c2018-05-10 15:44:45 +0300524 total_found += add_new_free_space(block_group, last,
Josef Bacik817d52f2009-07-13 21:29:25 -0400525 block_group->key.objectid +
526 block_group->key.offset);
Yan Zheng11833d62009-09-11 16:11:19 -0400527 caching_ctl->progress = (u64)-1;
Josef Bacik817d52f2009-07-13 21:29:25 -0400528
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700529out:
530 btrfs_free_path(path);
531 return ret;
532}
533
534static noinline void caching_thread(struct btrfs_work *work)
535{
536 struct btrfs_block_group_cache *block_group;
537 struct btrfs_fs_info *fs_info;
538 struct btrfs_caching_control *caching_ctl;
539 int ret;
540
541 caching_ctl = container_of(work, struct btrfs_caching_control, work);
542 block_group = caching_ctl->block_group;
543 fs_info = block_group->fs_info;
544
545 mutex_lock(&caching_ctl->mutex);
546 down_read(&fs_info->commit_root_sem);
547
Omar Sandoval1e144fb2015-09-29 20:50:37 -0700548 if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
549 ret = load_free_space_tree(caching_ctl);
550 else
551 ret = load_extent_tree_free(caching_ctl);
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700552
Josef Bacik817d52f2009-07-13 21:29:25 -0400553 spin_lock(&block_group->lock);
Yan Zheng11833d62009-09-11 16:11:19 -0400554 block_group->caching_ctl = NULL;
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700555 block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
Josef Bacik817d52f2009-07-13 21:29:25 -0400556 spin_unlock(&block_group->lock);
557
Josef Bacikd0bd4562015-09-23 14:54:14 -0400558#ifdef CONFIG_BTRFS_DEBUG
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400559 if (btrfs_should_fragment_free_space(block_group)) {
Josef Bacikd0bd4562015-09-23 14:54:14 -0400560 u64 bytes_used;
561
562 spin_lock(&block_group->space_info->lock);
563 spin_lock(&block_group->lock);
564 bytes_used = block_group->key.offset -
565 btrfs_block_group_used(&block_group->item);
566 block_group->space_info->bytes_used += bytes_used >> 1;
567 spin_unlock(&block_group->lock);
568 spin_unlock(&block_group->space_info->lock);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400569 fragment_free_space(block_group);
Josef Bacikd0bd4562015-09-23 14:54:14 -0400570 }
571#endif
572
573 caching_ctl->progress = (u64)-1;
Chris Masonf7d3d2f2015-12-18 11:11:10 -0800574
Josef Bacik9e351cc2014-03-13 15:42:13 -0400575 up_read(&fs_info->commit_root_sem);
Nikolay Borisov9e715da2018-06-20 15:49:08 +0300576 free_excluded_extents(block_group);
Yan Zheng11833d62009-09-11 16:11:19 -0400577 mutex_unlock(&caching_ctl->mutex);
Omar Sandoval73fa48b2015-09-29 20:50:33 -0700578
Yan Zheng11833d62009-09-11 16:11:19 -0400579 wake_up(&caching_ctl->wait);
580
581 put_caching_control(caching_ctl);
Josef Bacik11dfe352009-11-13 20:12:59 +0000582 btrfs_put_block_group(block_group);
Josef Bacik817d52f2009-07-13 21:29:25 -0400583}
584
Josef Bacik9d66e232010-08-25 16:54:15 -0400585static int cache_block_group(struct btrfs_block_group_cache *cache,
Josef Bacik9d66e232010-08-25 16:54:15 -0400586 int load_cache_only)
Josef Bacik817d52f2009-07-13 21:29:25 -0400587{
Josef Bacik291c7d22011-11-14 13:52:14 -0500588 DEFINE_WAIT(wait);
Yan Zheng11833d62009-09-11 16:11:19 -0400589 struct btrfs_fs_info *fs_info = cache->fs_info;
590 struct btrfs_caching_control *caching_ctl;
Josef Bacik817d52f2009-07-13 21:29:25 -0400591 int ret = 0;
592
Josef Bacik291c7d22011-11-14 13:52:14 -0500593 caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100594 if (!caching_ctl)
595 return -ENOMEM;
Josef Bacik291c7d22011-11-14 13:52:14 -0500596
597 INIT_LIST_HEAD(&caching_ctl->list);
598 mutex_init(&caching_ctl->mutex);
599 init_waitqueue_head(&caching_ctl->wait);
600 caching_ctl->block_group = cache;
601 caching_ctl->progress = cache->key.objectid;
Elena Reshetova1e4f4712017-03-03 10:55:14 +0200602 refcount_set(&caching_ctl->count, 1);
Liu Bo9e0af232014-08-15 23:36:53 +0800603 btrfs_init_work(&caching_ctl->work, btrfs_cache_helper,
604 caching_thread, NULL, NULL);
Josef Bacik291c7d22011-11-14 13:52:14 -0500605
606 spin_lock(&cache->lock);
607 /*
608 * This should be a rare occasion, but this could happen I think in the
609 * case where one thread starts to load the space cache info, and then
610 * some other thread starts a transaction commit which tries to do an
611 * allocation while the other thread is still loading the space cache
612 * info. The previous loop should have kept us from choosing this block
613 * group, but if we've moved to the state where we will wait on caching
614 * block groups we need to first check if we're doing a fast load here,
615 * so we can wait for it to finish, otherwise we could end up allocating
616 * from a block group who's cache gets evicted for one reason or
617 * another.
618 */
619 while (cache->cached == BTRFS_CACHE_FAST) {
620 struct btrfs_caching_control *ctl;
621
622 ctl = cache->caching_ctl;
Elena Reshetova1e4f4712017-03-03 10:55:14 +0200623 refcount_inc(&ctl->count);
Josef Bacik291c7d22011-11-14 13:52:14 -0500624 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
625 spin_unlock(&cache->lock);
626
627 schedule();
628
629 finish_wait(&ctl->wait, &wait);
630 put_caching_control(ctl);
631 spin_lock(&cache->lock);
632 }
633
634 if (cache->cached != BTRFS_CACHE_NO) {
635 spin_unlock(&cache->lock);
636 kfree(caching_ctl);
Yan Zheng11833d62009-09-11 16:11:19 -0400637 return 0;
Josef Bacik291c7d22011-11-14 13:52:14 -0500638 }
639 WARN_ON(cache->caching_ctl);
640 cache->caching_ctl = caching_ctl;
641 cache->cached = BTRFS_CACHE_FAST;
642 spin_unlock(&cache->lock);
Yan Zheng11833d62009-09-11 16:11:19 -0400643
Satoru Takeuchid8953d62017-09-12 20:08:08 +0900644 if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
Josef Bacikcb83b7b2014-11-26 11:52:54 -0500645 mutex_lock(&caching_ctl->mutex);
David Sterbabb6cb1c2019-03-20 13:47:15 +0100646 ret = load_free_space_cache(cache);
Josef Bacik9d66e232010-08-25 16:54:15 -0400647
648 spin_lock(&cache->lock);
649 if (ret == 1) {
Josef Bacik291c7d22011-11-14 13:52:14 -0500650 cache->caching_ctl = NULL;
Josef Bacik9d66e232010-08-25 16:54:15 -0400651 cache->cached = BTRFS_CACHE_FINISHED;
652 cache->last_byte_to_unpin = (u64)-1;
Josef Bacikcb83b7b2014-11-26 11:52:54 -0500653 caching_ctl->progress = (u64)-1;
Josef Bacik9d66e232010-08-25 16:54:15 -0400654 } else {
Josef Bacik291c7d22011-11-14 13:52:14 -0500655 if (load_cache_only) {
656 cache->caching_ctl = NULL;
657 cache->cached = BTRFS_CACHE_NO;
658 } else {
659 cache->cached = BTRFS_CACHE_STARTED;
Filipe Manana4f69cb92014-11-26 15:28:51 +0000660 cache->has_caching_ctl = 1;
Josef Bacik291c7d22011-11-14 13:52:14 -0500661 }
Josef Bacik9d66e232010-08-25 16:54:15 -0400662 }
663 spin_unlock(&cache->lock);
Josef Bacikd0bd4562015-09-23 14:54:14 -0400664#ifdef CONFIG_BTRFS_DEBUG
665 if (ret == 1 &&
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400666 btrfs_should_fragment_free_space(cache)) {
Josef Bacikd0bd4562015-09-23 14:54:14 -0400667 u64 bytes_used;
668
669 spin_lock(&cache->space_info->lock);
670 spin_lock(&cache->lock);
671 bytes_used = cache->key.offset -
672 btrfs_block_group_used(&cache->item);
673 cache->space_info->bytes_used += bytes_used >> 1;
674 spin_unlock(&cache->lock);
675 spin_unlock(&cache->space_info->lock);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400676 fragment_free_space(cache);
Josef Bacikd0bd4562015-09-23 14:54:14 -0400677 }
678#endif
Josef Bacikcb83b7b2014-11-26 11:52:54 -0500679 mutex_unlock(&caching_ctl->mutex);
680
Josef Bacik291c7d22011-11-14 13:52:14 -0500681 wake_up(&caching_ctl->wait);
Josef Bacik3c148742011-02-02 15:53:47 +0000682 if (ret == 1) {
Josef Bacik291c7d22011-11-14 13:52:14 -0500683 put_caching_control(caching_ctl);
Nikolay Borisov9e715da2018-06-20 15:49:08 +0300684 free_excluded_extents(cache);
Josef Bacik9d66e232010-08-25 16:54:15 -0400685 return 0;
Josef Bacik3c148742011-02-02 15:53:47 +0000686 }
Josef Bacik291c7d22011-11-14 13:52:14 -0500687 } else {
688 /*
Omar Sandoval1e144fb2015-09-29 20:50:37 -0700689 * We're either using the free space tree or no caching at all.
690 * Set cached to the appropriate value and wakeup any waiters.
Josef Bacik291c7d22011-11-14 13:52:14 -0500691 */
692 spin_lock(&cache->lock);
693 if (load_cache_only) {
694 cache->caching_ctl = NULL;
695 cache->cached = BTRFS_CACHE_NO;
696 } else {
697 cache->cached = BTRFS_CACHE_STARTED;
Filipe Manana4f69cb92014-11-26 15:28:51 +0000698 cache->has_caching_ctl = 1;
Josef Bacik291c7d22011-11-14 13:52:14 -0500699 }
Josef Bacik817d52f2009-07-13 21:29:25 -0400700 spin_unlock(&cache->lock);
Josef Bacik291c7d22011-11-14 13:52:14 -0500701 wake_up(&caching_ctl->wait);
702 }
703
704 if (load_cache_only) {
705 put_caching_control(caching_ctl);
Yan Zheng11833d62009-09-11 16:11:19 -0400706 return 0;
Josef Bacik817d52f2009-07-13 21:29:25 -0400707 }
Josef Bacik817d52f2009-07-13 21:29:25 -0400708
Josef Bacik9e351cc2014-03-13 15:42:13 -0400709 down_write(&fs_info->commit_root_sem);
Elena Reshetova1e4f4712017-03-03 10:55:14 +0200710 refcount_inc(&caching_ctl->count);
Yan Zheng11833d62009-09-11 16:11:19 -0400711 list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
Josef Bacik9e351cc2014-03-13 15:42:13 -0400712 up_write(&fs_info->commit_root_sem);
Yan Zheng11833d62009-09-11 16:11:19 -0400713
Josef Bacik11dfe352009-11-13 20:12:59 +0000714 btrfs_get_block_group(cache);
Yan Zheng11833d62009-09-11 16:11:19 -0400715
Qu Wenruoe66f0bb2014-02-28 10:46:12 +0800716 btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work);
Josef Bacik817d52f2009-07-13 21:29:25 -0400717
Josef Bacikef8bbdf2008-09-23 13:14:11 -0400718 return ret;
Chris Masone37c9e62007-05-09 20:13:14 -0400719}
720
Josef Bacik0f9dd462008-09-23 13:14:11 -0400721/*
722 * return the block group that starts at or after bytenr
723 */
Chris Masond3977122009-01-05 21:25:51 -0500724static struct btrfs_block_group_cache *
725btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
Chris Mason0ef3e662008-05-24 14:04:53 -0400726{
Masahiro Yamadae2c89902016-09-13 04:35:52 +0900727 return block_group_cache_tree_search(info, bytenr, 0);
Chris Mason0ef3e662008-05-24 14:04:53 -0400728}
729
Josef Bacik0f9dd462008-09-23 13:14:11 -0400730/*
Sankar P9f556842009-05-14 13:52:22 -0400731 * return the block group that contains the given bytenr
Josef Bacik0f9dd462008-09-23 13:14:11 -0400732 */
Chris Masond3977122009-01-05 21:25:51 -0500733struct btrfs_block_group_cache *btrfs_lookup_block_group(
734 struct btrfs_fs_info *info,
735 u64 bytenr)
Chris Masonbe744172007-05-06 10:15:01 -0400736{
Masahiro Yamadae2c89902016-09-13 04:35:52 +0900737 return block_group_cache_tree_search(info, bytenr, 1);
Chris Masonbe744172007-05-06 10:15:01 -0400738}
Chris Mason0b86a832008-03-24 15:01:56 -0400739
Josef Bacik0f9dd462008-09-23 13:14:11 -0400740static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
741 u64 flags)
Chris Mason6324fbf2008-03-24 15:01:59 -0400742{
Josef Bacik0f9dd462008-09-23 13:14:11 -0400743 struct list_head *head = &info->space_info;
Josef Bacik0f9dd462008-09-23 13:14:11 -0400744 struct btrfs_space_info *found;
Chris Mason4184ea72009-03-10 12:39:20 -0400745
Ilya Dryomov52ba6922012-01-16 22:04:47 +0200746 flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
Yan, Zhengb742bb822010-05-16 10:46:24 -0400747
Chris Mason4184ea72009-03-10 12:39:20 -0400748 rcu_read_lock();
749 list_for_each_entry_rcu(found, head, list) {
Josef Bacik67377732010-09-16 16:19:09 -0400750 if (found->flags & flags) {
Chris Mason4184ea72009-03-10 12:39:20 -0400751 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -0400752 return found;
Chris Mason4184ea72009-03-10 12:39:20 -0400753 }
Josef Bacik0f9dd462008-09-23 13:14:11 -0400754 }
Chris Mason4184ea72009-03-10 12:39:20 -0400755 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -0400756 return NULL;
Chris Mason6324fbf2008-03-24 15:01:59 -0400757}
758
Omar Sandoval0d9f8242017-06-06 16:45:26 -0700759static void add_pinned_bytes(struct btrfs_fs_info *fs_info, s64 num_bytes,
Nikolay Borisov29d2b842018-03-30 12:58:47 +0300760 bool metadata, u64 root_objectid)
Omar Sandoval0d9f8242017-06-06 16:45:26 -0700761{
762 struct btrfs_space_info *space_info;
763 u64 flags;
764
Nikolay Borisov29d2b842018-03-30 12:58:47 +0300765 if (metadata) {
Omar Sandoval0d9f8242017-06-06 16:45:26 -0700766 if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID)
767 flags = BTRFS_BLOCK_GROUP_SYSTEM;
768 else
769 flags = BTRFS_BLOCK_GROUP_METADATA;
770 } else {
771 flags = BTRFS_BLOCK_GROUP_DATA;
772 }
773
774 space_info = __find_space_info(fs_info, flags);
Omar Sandoval55e81962017-06-06 16:45:27 -0700775 ASSERT(space_info);
Ethan Liendec59fa2018-07-13 16:50:42 +0800776 percpu_counter_add_batch(&space_info->total_bytes_pinned, num_bytes,
777 BTRFS_TOTAL_BYTES_PINNED_BATCH);
Omar Sandoval0d9f8242017-06-06 16:45:26 -0700778}
779
Chris Mason4184ea72009-03-10 12:39:20 -0400780/*
781 * after adding space to the filesystem, we need to clear the full flags
782 * on all the space infos.
783 */
784void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
785{
786 struct list_head *head = &info->space_info;
787 struct btrfs_space_info *found;
788
789 rcu_read_lock();
790 list_for_each_entry_rcu(found, head, list)
791 found->full = 0;
792 rcu_read_unlock();
793}
794
Filipe Manana1a4ed8f2014-10-27 10:44:24 +0000795/* simple helper to search for an existing data extent at a given offset */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400796int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len)
Chris Masone02119d2008-09-05 16:13:11 -0400797{
798 int ret;
799 struct btrfs_key key;
Zheng Yan31840ae2008-09-23 13:14:14 -0400800 struct btrfs_path *path;
Chris Masone02119d2008-09-05 16:13:11 -0400801
Zheng Yan31840ae2008-09-23 13:14:14 -0400802 path = btrfs_alloc_path();
Mark Fashehd8926bb2011-07-13 10:38:47 -0700803 if (!path)
804 return -ENOMEM;
805
Chris Masone02119d2008-09-05 16:13:11 -0400806 key.objectid = start;
807 key.offset = len;
Josef Bacik3173a182013-03-07 14:22:04 -0500808 key.type = BTRFS_EXTENT_ITEM_KEY;
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400809 ret = btrfs_search_slot(NULL, fs_info->extent_root, &key, path, 0, 0);
Zheng Yan31840ae2008-09-23 13:14:14 -0400810 btrfs_free_path(path);
Chris Mason7bb86312007-12-11 09:25:06 -0500811 return ret;
812}
813
Chris Masond8d5f3e2007-12-11 12:42:00 -0500814/*
Josef Bacik3173a182013-03-07 14:22:04 -0500815 * helper function to lookup reference count and flags of a tree block.
Yan, Zhenga22285a2010-05-16 10:48:46 -0400816 *
817 * the head node for delayed ref is used to store the sum of all the
818 * reference count modifications queued up in the rbtree. the head
819 * node may also store the extent flags to set. This way you can check
820 * to see what the reference count and extent flags would be if all of
821 * the delayed refs are not processed.
822 */
823int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -0400824 struct btrfs_fs_info *fs_info, u64 bytenr,
Josef Bacik3173a182013-03-07 14:22:04 -0500825 u64 offset, int metadata, u64 *refs, u64 *flags)
Yan, Zhenga22285a2010-05-16 10:48:46 -0400826{
827 struct btrfs_delayed_ref_head *head;
828 struct btrfs_delayed_ref_root *delayed_refs;
829 struct btrfs_path *path;
830 struct btrfs_extent_item *ei;
831 struct extent_buffer *leaf;
832 struct btrfs_key key;
833 u32 item_size;
834 u64 num_refs;
835 u64 extent_flags;
836 int ret;
837
Josef Bacik3173a182013-03-07 14:22:04 -0500838 /*
839 * If we don't have skinny metadata, don't bother doing anything
840 * different
841 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400842 if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) {
843 offset = fs_info->nodesize;
Josef Bacik3173a182013-03-07 14:22:04 -0500844 metadata = 0;
845 }
846
Yan, Zhenga22285a2010-05-16 10:48:46 -0400847 path = btrfs_alloc_path();
848 if (!path)
849 return -ENOMEM;
850
Yan, Zhenga22285a2010-05-16 10:48:46 -0400851 if (!trans) {
852 path->skip_locking = 1;
853 path->search_commit_root = 1;
854 }
Filipe David Borba Manana639eefc2013-12-08 00:26:29 +0000855
856search_again:
857 key.objectid = bytenr;
858 key.offset = offset;
859 if (metadata)
860 key.type = BTRFS_METADATA_ITEM_KEY;
861 else
862 key.type = BTRFS_EXTENT_ITEM_KEY;
863
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400864 ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 0);
Yan, Zhenga22285a2010-05-16 10:48:46 -0400865 if (ret < 0)
866 goto out_free;
867
Josef Bacik3173a182013-03-07 14:22:04 -0500868 if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) {
Filipe David Borba Manana74be9512013-07-05 23:12:06 +0100869 if (path->slots[0]) {
870 path->slots[0]--;
871 btrfs_item_key_to_cpu(path->nodes[0], &key,
872 path->slots[0]);
873 if (key.objectid == bytenr &&
874 key.type == BTRFS_EXTENT_ITEM_KEY &&
Jeff Mahoney0b246af2016-06-22 18:54:23 -0400875 key.offset == fs_info->nodesize)
Filipe David Borba Manana74be9512013-07-05 23:12:06 +0100876 ret = 0;
877 }
Josef Bacik3173a182013-03-07 14:22:04 -0500878 }
879
Yan, Zhenga22285a2010-05-16 10:48:46 -0400880 if (ret == 0) {
881 leaf = path->nodes[0];
882 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
883 if (item_size >= sizeof(*ei)) {
884 ei = btrfs_item_ptr(leaf, path->slots[0],
885 struct btrfs_extent_item);
886 num_refs = btrfs_extent_refs(leaf, ei);
887 extent_flags = btrfs_extent_flags(leaf, ei);
888 } else {
Nikolay Borisovba3c2b12018-06-26 16:57:36 +0300889 ret = -EINVAL;
890 btrfs_print_v0_err(fs_info);
891 if (trans)
892 btrfs_abort_transaction(trans, ret);
893 else
894 btrfs_handle_fs_error(fs_info, ret, NULL);
895
896 goto out_free;
Yan, Zhenga22285a2010-05-16 10:48:46 -0400897 }
Nikolay Borisovba3c2b12018-06-26 16:57:36 +0300898
Yan, Zhenga22285a2010-05-16 10:48:46 -0400899 BUG_ON(num_refs == 0);
900 } else {
901 num_refs = 0;
902 extent_flags = 0;
903 ret = 0;
904 }
905
906 if (!trans)
907 goto out;
908
909 delayed_refs = &trans->transaction->delayed_refs;
910 spin_lock(&delayed_refs->lock);
Liu Bof72ad18e2017-01-30 12:24:37 -0800911 head = btrfs_find_delayed_ref_head(delayed_refs, bytenr);
Yan, Zhenga22285a2010-05-16 10:48:46 -0400912 if (head) {
913 if (!mutex_trylock(&head->mutex)) {
Josef Bacikd2788502017-09-29 15:43:57 -0400914 refcount_inc(&head->refs);
Yan, Zhenga22285a2010-05-16 10:48:46 -0400915 spin_unlock(&delayed_refs->lock);
916
David Sterbab3b4aa72011-04-21 01:20:15 +0200917 btrfs_release_path(path);
Yan, Zhenga22285a2010-05-16 10:48:46 -0400918
David Sterba8cc33e52011-05-02 15:29:25 +0200919 /*
920 * Mutex was contended, block until it's released and try
921 * again
922 */
Yan, Zhenga22285a2010-05-16 10:48:46 -0400923 mutex_lock(&head->mutex);
924 mutex_unlock(&head->mutex);
Josef Bacikd2788502017-09-29 15:43:57 -0400925 btrfs_put_delayed_ref_head(head);
Filipe David Borba Manana639eefc2013-12-08 00:26:29 +0000926 goto search_again;
Yan, Zhenga22285a2010-05-16 10:48:46 -0400927 }
Josef Bacikd7df2c72014-01-23 09:21:38 -0500928 spin_lock(&head->lock);
Yan, Zhenga22285a2010-05-16 10:48:46 -0400929 if (head->extent_op && head->extent_op->update_flags)
930 extent_flags |= head->extent_op->flags_to_set;
931 else
932 BUG_ON(num_refs == 0);
933
Josef Bacikd2788502017-09-29 15:43:57 -0400934 num_refs += head->ref_mod;
Josef Bacikd7df2c72014-01-23 09:21:38 -0500935 spin_unlock(&head->lock);
Yan, Zhenga22285a2010-05-16 10:48:46 -0400936 mutex_unlock(&head->mutex);
937 }
938 spin_unlock(&delayed_refs->lock);
939out:
940 WARN_ON(num_refs == 0);
941 if (refs)
942 *refs = num_refs;
943 if (flags)
944 *flags = extent_flags;
945out_free:
946 btrfs_free_path(path);
947 return ret;
948}
949
950/*
Chris Masond8d5f3e2007-12-11 12:42:00 -0500951 * Back reference rules. Back refs have three main goals:
952 *
953 * 1) differentiate between all holders of references to an extent so that
954 * when a reference is dropped we can make sure it was a valid reference
955 * before freeing the extent.
956 *
957 * 2) Provide enough information to quickly find the holders of an extent
958 * if we notice a given block is corrupted or bad.
959 *
960 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
961 * maintenance. This is actually the same as #2, but with a slightly
962 * different use case.
963 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400964 * There are two kinds of back refs. The implicit back refs is optimized
965 * for pointers in non-shared tree blocks. For a given pointer in a block,
966 * back refs of this kind provide information about the block's owner tree
967 * and the pointer's key. These information allow us to find the block by
968 * b-tree searching. The full back refs is for pointers in tree blocks not
969 * referenced by their owner trees. The location of tree block is recorded
970 * in the back refs. Actually the full back refs is generic, and can be
971 * used in all cases the implicit back refs is used. The major shortcoming
972 * of the full back refs is its overhead. Every time a tree block gets
973 * COWed, we have to update back refs entry for all pointers in it.
974 *
975 * For a newly allocated tree block, we use implicit back refs for
976 * pointers in it. This means most tree related operations only involve
977 * implicit back refs. For a tree block created in old transaction, the
978 * only way to drop a reference to it is COW it. So we can detect the
979 * event that tree block loses its owner tree's reference and do the
980 * back refs conversion.
981 *
Nicholas D Steeves01327612016-05-19 21:18:45 -0400982 * When a tree block is COWed through a tree, there are four cases:
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400983 *
984 * The reference count of the block is one and the tree is the block's
985 * owner tree. Nothing to do in this case.
986 *
987 * The reference count of the block is one and the tree is not the
988 * block's owner tree. In this case, full back refs is used for pointers
989 * in the block. Remove these full back refs, add implicit back refs for
990 * every pointers in the new block.
991 *
992 * The reference count of the block is greater than one and the tree is
993 * the block's owner tree. In this case, implicit back refs is used for
994 * pointers in the block. Add full back refs for every pointers in the
995 * block, increase lower level extents' reference counts. The original
996 * implicit back refs are entailed to the new block.
997 *
998 * The reference count of the block is greater than one and the tree is
999 * not the block's owner tree. Add implicit back refs for every pointer in
1000 * the new block, increase lower level extents' reference count.
1001 *
1002 * Back Reference Key composing:
1003 *
1004 * The key objectid corresponds to the first byte in the extent,
1005 * The key type is used to differentiate between types of back refs.
1006 * There are different meanings of the key offset for different types
1007 * of back refs.
1008 *
Chris Masond8d5f3e2007-12-11 12:42:00 -05001009 * File extents can be referenced by:
1010 *
1011 * - multiple snapshots, subvolumes, or different generations in one subvol
Zheng Yan31840ae2008-09-23 13:14:14 -04001012 * - different files inside a single subvolume
Chris Masond8d5f3e2007-12-11 12:42:00 -05001013 * - different offsets inside a file (bookend extents in file.c)
1014 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001015 * The extent ref structure for the implicit back refs has fields for:
Chris Masond8d5f3e2007-12-11 12:42:00 -05001016 *
1017 * - Objectid of the subvolume root
Chris Masond8d5f3e2007-12-11 12:42:00 -05001018 * - objectid of the file holding the reference
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001019 * - original offset in the file
1020 * - how many bookend extents
Zheng Yan31840ae2008-09-23 13:14:14 -04001021 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001022 * The key offset for the implicit back refs is hash of the first
1023 * three fields.
Chris Masond8d5f3e2007-12-11 12:42:00 -05001024 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001025 * The extent ref structure for the full back refs has field for:
Chris Masond8d5f3e2007-12-11 12:42:00 -05001026 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001027 * - number of pointers in the tree leaf
Chris Masond8d5f3e2007-12-11 12:42:00 -05001028 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001029 * The key offset for the implicit back refs is the first byte of
1030 * the tree leaf
Chris Masond8d5f3e2007-12-11 12:42:00 -05001031 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001032 * When a file extent is allocated, The implicit back refs is used.
1033 * the fields are filled in:
Chris Masond8d5f3e2007-12-11 12:42:00 -05001034 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001035 * (root_key.objectid, inode objectid, offset in file, 1)
1036 *
1037 * When a file extent is removed file truncation, we find the
1038 * corresponding implicit back refs and check the following fields:
1039 *
1040 * (btrfs_header_owner(leaf), inode objectid, offset in file)
Chris Masond8d5f3e2007-12-11 12:42:00 -05001041 *
1042 * Btree extents can be referenced by:
1043 *
1044 * - Different subvolumes
Chris Masond8d5f3e2007-12-11 12:42:00 -05001045 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001046 * Both the implicit back refs and the full back refs for tree blocks
1047 * only consist of key. The key offset for the implicit back refs is
1048 * objectid of block's owner tree. The key offset for the full back refs
1049 * is the first byte of parent block.
Chris Masond8d5f3e2007-12-11 12:42:00 -05001050 *
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001051 * When implicit back refs is used, information about the lowest key and
1052 * level of the tree block are required. These information are stored in
1053 * tree block info structure.
Chris Masond8d5f3e2007-12-11 12:42:00 -05001054 */
Zheng Yan31840ae2008-09-23 13:14:14 -04001055
Liu Bo167ce952017-08-18 15:15:18 -06001056/*
1057 * is_data == BTRFS_REF_TYPE_BLOCK, tree block type is required,
Andrea Gelmini52042d82018-11-28 12:05:13 +01001058 * is_data == BTRFS_REF_TYPE_DATA, data type is requiried,
Liu Bo167ce952017-08-18 15:15:18 -06001059 * is_data == BTRFS_REF_TYPE_ANY, either type is OK.
1060 */
1061int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb,
1062 struct btrfs_extent_inline_ref *iref,
1063 enum btrfs_inline_ref_type is_data)
1064{
1065 int type = btrfs_extent_inline_ref_type(eb, iref);
Liu Bo64ecdb62017-08-18 15:15:24 -06001066 u64 offset = btrfs_extent_inline_ref_offset(eb, iref);
Liu Bo167ce952017-08-18 15:15:18 -06001067
1068 if (type == BTRFS_TREE_BLOCK_REF_KEY ||
1069 type == BTRFS_SHARED_BLOCK_REF_KEY ||
1070 type == BTRFS_SHARED_DATA_REF_KEY ||
1071 type == BTRFS_EXTENT_DATA_REF_KEY) {
1072 if (is_data == BTRFS_REF_TYPE_BLOCK) {
Liu Bo64ecdb62017-08-18 15:15:24 -06001073 if (type == BTRFS_TREE_BLOCK_REF_KEY)
Liu Bo167ce952017-08-18 15:15:18 -06001074 return type;
Liu Bo64ecdb62017-08-18 15:15:24 -06001075 if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
1076 ASSERT(eb->fs_info);
1077 /*
1078 * Every shared one has parent tree
1079 * block, which must be aligned to
1080 * nodesize.
1081 */
1082 if (offset &&
1083 IS_ALIGNED(offset, eb->fs_info->nodesize))
1084 return type;
1085 }
Liu Bo167ce952017-08-18 15:15:18 -06001086 } else if (is_data == BTRFS_REF_TYPE_DATA) {
Liu Bo64ecdb62017-08-18 15:15:24 -06001087 if (type == BTRFS_EXTENT_DATA_REF_KEY)
Liu Bo167ce952017-08-18 15:15:18 -06001088 return type;
Liu Bo64ecdb62017-08-18 15:15:24 -06001089 if (type == BTRFS_SHARED_DATA_REF_KEY) {
1090 ASSERT(eb->fs_info);
1091 /*
1092 * Every shared one has parent tree
1093 * block, which must be aligned to
1094 * nodesize.
1095 */
1096 if (offset &&
1097 IS_ALIGNED(offset, eb->fs_info->nodesize))
1098 return type;
1099 }
Liu Bo167ce952017-08-18 15:15:18 -06001100 } else {
1101 ASSERT(is_data == BTRFS_REF_TYPE_ANY);
1102 return type;
1103 }
1104 }
1105
1106 btrfs_print_leaf((struct extent_buffer *)eb);
1107 btrfs_err(eb->fs_info, "eb %llu invalid extent inline ref type %d",
1108 eb->start, type);
1109 WARN_ON(1);
1110
1111 return BTRFS_REF_TYPE_INVALID;
1112}
1113
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001114static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
1115{
1116 u32 high_crc = ~(u32)0;
1117 u32 low_crc = ~(u32)0;
1118 __le64 lenum;
1119
1120 lenum = cpu_to_le64(root_objectid);
Nikolay Borisov9678c542018-01-08 11:45:05 +02001121 high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001122 lenum = cpu_to_le64(owner);
Nikolay Borisov9678c542018-01-08 11:45:05 +02001123 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001124 lenum = cpu_to_le64(offset);
Nikolay Borisov9678c542018-01-08 11:45:05 +02001125 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001126
1127 return ((u64)high_crc << 31) ^ (u64)low_crc;
1128}
1129
1130static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
1131 struct btrfs_extent_data_ref *ref)
1132{
1133 return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
1134 btrfs_extent_data_ref_objectid(leaf, ref),
1135 btrfs_extent_data_ref_offset(leaf, ref));
1136}
1137
1138static int match_extent_data_ref(struct extent_buffer *leaf,
1139 struct btrfs_extent_data_ref *ref,
1140 u64 root_objectid, u64 owner, u64 offset)
1141{
1142 if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
1143 btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
1144 btrfs_extent_data_ref_offset(leaf, ref) != offset)
1145 return 0;
1146 return 1;
1147}
1148
1149static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001150 struct btrfs_path *path,
1151 u64 bytenr, u64 parent,
1152 u64 root_objectid,
1153 u64 owner, u64 offset)
1154{
Nikolay Borisovbd1d53e2018-06-20 15:48:51 +03001155 struct btrfs_root *root = trans->fs_info->extent_root;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001156 struct btrfs_key key;
1157 struct btrfs_extent_data_ref *ref;
1158 struct extent_buffer *leaf;
1159 u32 nritems;
1160 int ret;
1161 int recow;
1162 int err = -ENOENT;
1163
1164 key.objectid = bytenr;
1165 if (parent) {
1166 key.type = BTRFS_SHARED_DATA_REF_KEY;
1167 key.offset = parent;
1168 } else {
1169 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1170 key.offset = hash_extent_data_ref(root_objectid,
1171 owner, offset);
1172 }
1173again:
1174 recow = 0;
1175 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1176 if (ret < 0) {
1177 err = ret;
1178 goto fail;
1179 }
1180
1181 if (parent) {
1182 if (!ret)
1183 return 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001184 goto fail;
Zheng Yan31840ae2008-09-23 13:14:14 -04001185 }
1186
1187 leaf = path->nodes[0];
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001188 nritems = btrfs_header_nritems(leaf);
1189 while (1) {
1190 if (path->slots[0] >= nritems) {
1191 ret = btrfs_next_leaf(root, path);
1192 if (ret < 0)
1193 err = ret;
1194 if (ret)
1195 goto fail;
1196
1197 leaf = path->nodes[0];
1198 nritems = btrfs_header_nritems(leaf);
1199 recow = 1;
1200 }
1201
1202 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1203 if (key.objectid != bytenr ||
1204 key.type != BTRFS_EXTENT_DATA_REF_KEY)
1205 goto fail;
1206
1207 ref = btrfs_item_ptr(leaf, path->slots[0],
1208 struct btrfs_extent_data_ref);
1209
1210 if (match_extent_data_ref(leaf, ref, root_objectid,
1211 owner, offset)) {
1212 if (recow) {
David Sterbab3b4aa72011-04-21 01:20:15 +02001213 btrfs_release_path(path);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001214 goto again;
1215 }
1216 err = 0;
1217 break;
1218 }
1219 path->slots[0]++;
Zheng Yan31840ae2008-09-23 13:14:14 -04001220 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001221fail:
1222 return err;
Zheng Yan31840ae2008-09-23 13:14:14 -04001223}
1224
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001225static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001226 struct btrfs_path *path,
1227 u64 bytenr, u64 parent,
1228 u64 root_objectid, u64 owner,
1229 u64 offset, int refs_to_add)
Zheng Yan31840ae2008-09-23 13:14:14 -04001230{
Nikolay Borisov62b895a2018-06-20 15:48:44 +03001231 struct btrfs_root *root = trans->fs_info->extent_root;
Zheng Yan31840ae2008-09-23 13:14:14 -04001232 struct btrfs_key key;
1233 struct extent_buffer *leaf;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001234 u32 size;
Zheng Yan31840ae2008-09-23 13:14:14 -04001235 u32 num_refs;
1236 int ret;
1237
1238 key.objectid = bytenr;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001239 if (parent) {
1240 key.type = BTRFS_SHARED_DATA_REF_KEY;
1241 key.offset = parent;
1242 size = sizeof(struct btrfs_shared_data_ref);
Zheng Yan31840ae2008-09-23 13:14:14 -04001243 } else {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001244 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1245 key.offset = hash_extent_data_ref(root_objectid,
1246 owner, offset);
1247 size = sizeof(struct btrfs_extent_data_ref);
Zheng Yan31840ae2008-09-23 13:14:14 -04001248 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001249
1250 ret = btrfs_insert_empty_item(trans, root, path, &key, size);
1251 if (ret && ret != -EEXIST)
1252 goto fail;
1253
1254 leaf = path->nodes[0];
1255 if (parent) {
1256 struct btrfs_shared_data_ref *ref;
1257 ref = btrfs_item_ptr(leaf, path->slots[0],
1258 struct btrfs_shared_data_ref);
1259 if (ret == 0) {
1260 btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
1261 } else {
1262 num_refs = btrfs_shared_data_ref_count(leaf, ref);
1263 num_refs += refs_to_add;
1264 btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
1265 }
1266 } else {
1267 struct btrfs_extent_data_ref *ref;
1268 while (ret == -EEXIST) {
1269 ref = btrfs_item_ptr(leaf, path->slots[0],
1270 struct btrfs_extent_data_ref);
1271 if (match_extent_data_ref(leaf, ref, root_objectid,
1272 owner, offset))
1273 break;
David Sterbab3b4aa72011-04-21 01:20:15 +02001274 btrfs_release_path(path);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001275 key.offset++;
1276 ret = btrfs_insert_empty_item(trans, root, path, &key,
1277 size);
1278 if (ret && ret != -EEXIST)
1279 goto fail;
1280
1281 leaf = path->nodes[0];
1282 }
1283 ref = btrfs_item_ptr(leaf, path->slots[0],
1284 struct btrfs_extent_data_ref);
1285 if (ret == 0) {
1286 btrfs_set_extent_data_ref_root(leaf, ref,
1287 root_objectid);
1288 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
1289 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
1290 btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
1291 } else {
1292 num_refs = btrfs_extent_data_ref_count(leaf, ref);
1293 num_refs += refs_to_add;
1294 btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
1295 }
1296 }
1297 btrfs_mark_buffer_dirty(leaf);
1298 ret = 0;
1299fail:
David Sterbab3b4aa72011-04-21 01:20:15 +02001300 btrfs_release_path(path);
Chris Mason7bb86312007-12-11 09:25:06 -05001301 return ret;
Chris Mason74493f72007-12-11 09:25:06 -05001302}
1303
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001304static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001305 struct btrfs_path *path,
Josef Bacikfcebe452014-05-13 17:30:47 -07001306 int refs_to_drop, int *last_ref)
Zheng Yan31840ae2008-09-23 13:14:14 -04001307{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001308 struct btrfs_key key;
1309 struct btrfs_extent_data_ref *ref1 = NULL;
1310 struct btrfs_shared_data_ref *ref2 = NULL;
Zheng Yan31840ae2008-09-23 13:14:14 -04001311 struct extent_buffer *leaf;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001312 u32 num_refs = 0;
Zheng Yan31840ae2008-09-23 13:14:14 -04001313 int ret = 0;
1314
1315 leaf = path->nodes[0];
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001316 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1317
1318 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1319 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1320 struct btrfs_extent_data_ref);
1321 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1322 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1323 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1324 struct btrfs_shared_data_ref);
1325 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
David Sterba6d8ff4e2018-06-26 16:20:59 +02001326 } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) {
Nikolay Borisovba3c2b12018-06-26 16:57:36 +03001327 btrfs_print_v0_err(trans->fs_info);
1328 btrfs_abort_transaction(trans, -EINVAL);
1329 return -EINVAL;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001330 } else {
1331 BUG();
1332 }
1333
Chris Mason56bec292009-03-13 10:10:06 -04001334 BUG_ON(num_refs < refs_to_drop);
1335 num_refs -= refs_to_drop;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001336
Zheng Yan31840ae2008-09-23 13:14:14 -04001337 if (num_refs == 0) {
Nikolay Borisove9f62902018-06-20 15:48:46 +03001338 ret = btrfs_del_item(trans, trans->fs_info->extent_root, path);
Josef Bacikfcebe452014-05-13 17:30:47 -07001339 *last_ref = 1;
Zheng Yan31840ae2008-09-23 13:14:14 -04001340 } else {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001341 if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
1342 btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
1343 else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
1344 btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
Zheng Yan31840ae2008-09-23 13:14:14 -04001345 btrfs_mark_buffer_dirty(leaf);
1346 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001347 return ret;
1348}
1349
Zhaolei9ed0dea2015-08-06 22:16:24 +08001350static noinline u32 extent_data_ref_count(struct btrfs_path *path,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001351 struct btrfs_extent_inline_ref *iref)
1352{
1353 struct btrfs_key key;
1354 struct extent_buffer *leaf;
1355 struct btrfs_extent_data_ref *ref1;
1356 struct btrfs_shared_data_ref *ref2;
1357 u32 num_refs = 0;
Liu Bo3de28d52017-08-18 15:15:19 -06001358 int type;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001359
1360 leaf = path->nodes[0];
1361 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
Nikolay Borisovba3c2b12018-06-26 16:57:36 +03001362
1363 BUG_ON(key.type == BTRFS_EXTENT_REF_V0_KEY);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001364 if (iref) {
Liu Bo3de28d52017-08-18 15:15:19 -06001365 /*
1366 * If type is invalid, we should have bailed out earlier than
1367 * this call.
1368 */
1369 type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA);
1370 ASSERT(type != BTRFS_REF_TYPE_INVALID);
1371 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001372 ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
1373 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1374 } else {
1375 ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
1376 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1377 }
1378 } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1379 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1380 struct btrfs_extent_data_ref);
1381 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1382 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1383 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1384 struct btrfs_shared_data_ref);
1385 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001386 } else {
1387 WARN_ON(1);
1388 }
1389 return num_refs;
1390}
1391
1392static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001393 struct btrfs_path *path,
1394 u64 bytenr, u64 parent,
1395 u64 root_objectid)
1396{
Nikolay Borisovb8582ee2018-06-20 15:48:50 +03001397 struct btrfs_root *root = trans->fs_info->extent_root;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001398 struct btrfs_key key;
1399 int ret;
1400
1401 key.objectid = bytenr;
1402 if (parent) {
1403 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1404 key.offset = parent;
1405 } else {
1406 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1407 key.offset = root_objectid;
1408 }
1409
1410 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1411 if (ret > 0)
1412 ret = -ENOENT;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001413 return ret;
1414}
1415
1416static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001417 struct btrfs_path *path,
1418 u64 bytenr, u64 parent,
1419 u64 root_objectid)
1420{
1421 struct btrfs_key key;
1422 int ret;
1423
1424 key.objectid = bytenr;
1425 if (parent) {
1426 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1427 key.offset = parent;
1428 } else {
1429 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1430 key.offset = root_objectid;
1431 }
1432
Nikolay Borisov10728402018-06-20 15:48:43 +03001433 ret = btrfs_insert_empty_item(trans, trans->fs_info->extent_root,
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05001434 path, &key, 0);
David Sterbab3b4aa72011-04-21 01:20:15 +02001435 btrfs_release_path(path);
Zheng Yan31840ae2008-09-23 13:14:14 -04001436 return ret;
1437}
1438
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001439static inline int extent_ref_type(u64 parent, u64 owner)
1440{
1441 int type;
1442 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1443 if (parent > 0)
1444 type = BTRFS_SHARED_BLOCK_REF_KEY;
1445 else
1446 type = BTRFS_TREE_BLOCK_REF_KEY;
1447 } else {
1448 if (parent > 0)
1449 type = BTRFS_SHARED_DATA_REF_KEY;
1450 else
1451 type = BTRFS_EXTENT_DATA_REF_KEY;
1452 }
1453 return type;
1454}
1455
Yan Zheng2c47e6052009-06-27 21:07:35 -04001456static int find_next_key(struct btrfs_path *path, int level,
1457 struct btrfs_key *key)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001458
1459{
Yan Zheng2c47e6052009-06-27 21:07:35 -04001460 for (; level < BTRFS_MAX_LEVEL; level++) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001461 if (!path->nodes[level])
1462 break;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001463 if (path->slots[level] + 1 >=
1464 btrfs_header_nritems(path->nodes[level]))
1465 continue;
1466 if (level == 0)
1467 btrfs_item_key_to_cpu(path->nodes[level], key,
1468 path->slots[level] + 1);
1469 else
1470 btrfs_node_key_to_cpu(path->nodes[level], key,
1471 path->slots[level] + 1);
1472 return 0;
1473 }
1474 return 1;
1475}
1476
1477/*
1478 * look for inline back ref. if back ref is found, *ref_ret is set
1479 * to the address of inline back ref, and 0 is returned.
1480 *
1481 * if back ref isn't found, *ref_ret is set to the address where it
1482 * should be inserted, and -ENOENT is returned.
1483 *
1484 * if insert is true and there are too many inline back refs, the path
1485 * points to the extent item, and -EAGAIN is returned.
1486 *
1487 * NOTE: inline back refs are ordered in the same way that back ref
1488 * items in the tree are ordered.
1489 */
1490static noinline_for_stack
1491int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001492 struct btrfs_path *path,
1493 struct btrfs_extent_inline_ref **ref_ret,
1494 u64 bytenr, u64 num_bytes,
1495 u64 parent, u64 root_objectid,
1496 u64 owner, u64 offset, int insert)
1497{
Nikolay Borisov867cc1f2018-06-20 15:48:48 +03001498 struct btrfs_fs_info *fs_info = trans->fs_info;
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05001499 struct btrfs_root *root = fs_info->extent_root;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001500 struct btrfs_key key;
1501 struct extent_buffer *leaf;
1502 struct btrfs_extent_item *ei;
1503 struct btrfs_extent_inline_ref *iref;
1504 u64 flags;
1505 u64 item_size;
1506 unsigned long ptr;
1507 unsigned long end;
1508 int extra_size;
1509 int type;
1510 int want;
1511 int ret;
1512 int err = 0;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001513 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA);
Liu Bo3de28d52017-08-18 15:15:19 -06001514 int needed;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001515
1516 key.objectid = bytenr;
1517 key.type = BTRFS_EXTENT_ITEM_KEY;
1518 key.offset = num_bytes;
1519
1520 want = extent_ref_type(parent, owner);
1521 if (insert) {
1522 extra_size = btrfs_extent_inline_ref_size(want);
Yan Zheng85d41982009-06-11 08:51:10 -04001523 path->keep_locks = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001524 } else
1525 extra_size = -1;
Josef Bacik3173a182013-03-07 14:22:04 -05001526
1527 /*
Nikolay Borisov16d1c062018-06-18 14:59:26 +03001528 * Owner is our level, so we can just add one to get the level for the
1529 * block we are interested in.
Josef Bacik3173a182013-03-07 14:22:04 -05001530 */
1531 if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) {
1532 key.type = BTRFS_METADATA_ITEM_KEY;
1533 key.offset = owner;
1534 }
1535
1536again:
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001537 ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1538 if (ret < 0) {
1539 err = ret;
1540 goto out;
1541 }
Josef Bacik3173a182013-03-07 14:22:04 -05001542
1543 /*
1544 * We may be a newly converted file system which still has the old fat
1545 * extent entries for metadata, so try and see if we have one of those.
1546 */
1547 if (ret > 0 && skinny_metadata) {
1548 skinny_metadata = false;
1549 if (path->slots[0]) {
1550 path->slots[0]--;
1551 btrfs_item_key_to_cpu(path->nodes[0], &key,
1552 path->slots[0]);
1553 if (key.objectid == bytenr &&
1554 key.type == BTRFS_EXTENT_ITEM_KEY &&
1555 key.offset == num_bytes)
1556 ret = 0;
1557 }
1558 if (ret) {
Filipe Manana9ce49a02014-04-24 15:15:28 +01001559 key.objectid = bytenr;
Josef Bacik3173a182013-03-07 14:22:04 -05001560 key.type = BTRFS_EXTENT_ITEM_KEY;
1561 key.offset = num_bytes;
1562 btrfs_release_path(path);
1563 goto again;
1564 }
1565 }
1566
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001567 if (ret && !insert) {
1568 err = -ENOENT;
1569 goto out;
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301570 } else if (WARN_ON(ret)) {
Josef Bacik492104c2013-03-08 15:41:02 -05001571 err = -EIO;
Josef Bacik492104c2013-03-08 15:41:02 -05001572 goto out;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001573 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001574
1575 leaf = path->nodes[0];
1576 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
David Sterba6d8ff4e2018-06-26 16:20:59 +02001577 if (unlikely(item_size < sizeof(*ei))) {
Nikolay Borisovba3c2b12018-06-26 16:57:36 +03001578 err = -EINVAL;
1579 btrfs_print_v0_err(fs_info);
1580 btrfs_abort_transaction(trans, err);
1581 goto out;
1582 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001583
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001584 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1585 flags = btrfs_extent_flags(leaf, ei);
1586
1587 ptr = (unsigned long)(ei + 1);
1588 end = (unsigned long)ei + item_size;
1589
Josef Bacik3173a182013-03-07 14:22:04 -05001590 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001591 ptr += sizeof(struct btrfs_tree_block_info);
1592 BUG_ON(ptr > end);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001593 }
1594
Liu Bo3de28d52017-08-18 15:15:19 -06001595 if (owner >= BTRFS_FIRST_FREE_OBJECTID)
1596 needed = BTRFS_REF_TYPE_DATA;
1597 else
1598 needed = BTRFS_REF_TYPE_BLOCK;
1599
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001600 err = -ENOENT;
1601 while (1) {
1602 if (ptr >= end) {
1603 WARN_ON(ptr > end);
1604 break;
1605 }
1606 iref = (struct btrfs_extent_inline_ref *)ptr;
Liu Bo3de28d52017-08-18 15:15:19 -06001607 type = btrfs_get_extent_inline_ref_type(leaf, iref, needed);
1608 if (type == BTRFS_REF_TYPE_INVALID) {
Su Yueaf431dc2018-06-22 16:18:01 +08001609 err = -EUCLEAN;
Liu Bo3de28d52017-08-18 15:15:19 -06001610 goto out;
1611 }
1612
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001613 if (want < type)
1614 break;
1615 if (want > type) {
1616 ptr += btrfs_extent_inline_ref_size(type);
1617 continue;
1618 }
1619
1620 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1621 struct btrfs_extent_data_ref *dref;
1622 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1623 if (match_extent_data_ref(leaf, dref, root_objectid,
1624 owner, offset)) {
1625 err = 0;
1626 break;
1627 }
1628 if (hash_extent_data_ref_item(leaf, dref) <
1629 hash_extent_data_ref(root_objectid, owner, offset))
1630 break;
1631 } else {
1632 u64 ref_offset;
1633 ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
1634 if (parent > 0) {
1635 if (parent == ref_offset) {
1636 err = 0;
1637 break;
1638 }
1639 if (ref_offset < parent)
1640 break;
1641 } else {
1642 if (root_objectid == ref_offset) {
1643 err = 0;
1644 break;
1645 }
1646 if (ref_offset < root_objectid)
1647 break;
1648 }
1649 }
1650 ptr += btrfs_extent_inline_ref_size(type);
1651 }
1652 if (err == -ENOENT && insert) {
1653 if (item_size + extra_size >=
1654 BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
1655 err = -EAGAIN;
1656 goto out;
1657 }
1658 /*
1659 * To add new inline back ref, we have to make sure
1660 * there is no corresponding back ref item.
1661 * For simplicity, we just do not add new inline back
1662 * ref if there is any kind of item for this block
1663 */
Yan Zheng2c47e6052009-06-27 21:07:35 -04001664 if (find_next_key(path, 0, &key) == 0 &&
1665 key.objectid == bytenr &&
Yan Zheng85d41982009-06-11 08:51:10 -04001666 key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001667 err = -EAGAIN;
1668 goto out;
1669 }
1670 }
1671 *ref_ret = (struct btrfs_extent_inline_ref *)ptr;
1672out:
Yan Zheng85d41982009-06-11 08:51:10 -04001673 if (insert) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001674 path->keep_locks = 0;
1675 btrfs_unlock_up_safe(path, 1);
1676 }
1677 return err;
1678}
1679
1680/*
1681 * helper to add new inline back ref
1682 */
1683static noinline_for_stack
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05001684void setup_inline_extent_backref(struct btrfs_fs_info *fs_info,
Jeff Mahoney143bede2012-03-01 14:56:26 +01001685 struct btrfs_path *path,
1686 struct btrfs_extent_inline_ref *iref,
1687 u64 parent, u64 root_objectid,
1688 u64 owner, u64 offset, int refs_to_add,
1689 struct btrfs_delayed_extent_op *extent_op)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001690{
1691 struct extent_buffer *leaf;
1692 struct btrfs_extent_item *ei;
1693 unsigned long ptr;
1694 unsigned long end;
1695 unsigned long item_offset;
1696 u64 refs;
1697 int size;
1698 int type;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001699
1700 leaf = path->nodes[0];
1701 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1702 item_offset = (unsigned long)iref - (unsigned long)ei;
1703
1704 type = extent_ref_type(parent, owner);
1705 size = btrfs_extent_inline_ref_size(type);
1706
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05001707 btrfs_extend_item(fs_info, path, size);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001708
1709 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1710 refs = btrfs_extent_refs(leaf, ei);
1711 refs += refs_to_add;
1712 btrfs_set_extent_refs(leaf, ei, refs);
1713 if (extent_op)
1714 __run_delayed_extent_op(extent_op, leaf, ei);
1715
1716 ptr = (unsigned long)ei + item_offset;
1717 end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
1718 if (ptr < end - size)
1719 memmove_extent_buffer(leaf, ptr + size, ptr,
1720 end - size - ptr);
1721
1722 iref = (struct btrfs_extent_inline_ref *)ptr;
1723 btrfs_set_extent_inline_ref_type(leaf, iref, type);
1724 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1725 struct btrfs_extent_data_ref *dref;
1726 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1727 btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
1728 btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
1729 btrfs_set_extent_data_ref_offset(leaf, dref, offset);
1730 btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
1731 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1732 struct btrfs_shared_data_ref *sref;
1733 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1734 btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
1735 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1736 } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
1737 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1738 } else {
1739 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
1740 }
1741 btrfs_mark_buffer_dirty(leaf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001742}
1743
1744static int lookup_extent_backref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001745 struct btrfs_path *path,
1746 struct btrfs_extent_inline_ref **ref_ret,
1747 u64 bytenr, u64 num_bytes, u64 parent,
1748 u64 root_objectid, u64 owner, u64 offset)
1749{
1750 int ret;
1751
Nikolay Borisov867cc1f2018-06-20 15:48:48 +03001752 ret = lookup_inline_extent_backref(trans, path, ref_ret, bytenr,
1753 num_bytes, parent, root_objectid,
1754 owner, offset, 0);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001755 if (ret != -ENOENT)
1756 return ret;
1757
David Sterbab3b4aa72011-04-21 01:20:15 +02001758 btrfs_release_path(path);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001759 *ref_ret = NULL;
1760
1761 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
Nikolay Borisovb8582ee2018-06-20 15:48:50 +03001762 ret = lookup_tree_block_ref(trans, path, bytenr, parent,
1763 root_objectid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001764 } else {
Nikolay Borisovbd1d53e2018-06-20 15:48:51 +03001765 ret = lookup_extent_data_ref(trans, path, bytenr, parent,
1766 root_objectid, owner, offset);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001767 }
1768 return ret;
1769}
1770
1771/*
1772 * helper to update/remove inline back ref
1773 */
1774static noinline_for_stack
Nikolay Borisov61a18f12018-06-20 15:48:49 +03001775void update_inline_extent_backref(struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01001776 struct btrfs_extent_inline_ref *iref,
1777 int refs_to_mod,
Josef Bacikfcebe452014-05-13 17:30:47 -07001778 struct btrfs_delayed_extent_op *extent_op,
1779 int *last_ref)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001780{
Nikolay Borisov61a18f12018-06-20 15:48:49 +03001781 struct extent_buffer *leaf = path->nodes[0];
1782 struct btrfs_fs_info *fs_info = leaf->fs_info;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001783 struct btrfs_extent_item *ei;
1784 struct btrfs_extent_data_ref *dref = NULL;
1785 struct btrfs_shared_data_ref *sref = NULL;
1786 unsigned long ptr;
1787 unsigned long end;
1788 u32 item_size;
1789 int size;
1790 int type;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001791 u64 refs;
1792
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001793 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1794 refs = btrfs_extent_refs(leaf, ei);
1795 WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
1796 refs += refs_to_mod;
1797 btrfs_set_extent_refs(leaf, ei, refs);
1798 if (extent_op)
1799 __run_delayed_extent_op(extent_op, leaf, ei);
1800
Liu Bo3de28d52017-08-18 15:15:19 -06001801 /*
1802 * If type is invalid, we should have bailed out after
1803 * lookup_inline_extent_backref().
1804 */
1805 type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_ANY);
1806 ASSERT(type != BTRFS_REF_TYPE_INVALID);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001807
1808 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1809 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1810 refs = btrfs_extent_data_ref_count(leaf, dref);
1811 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1812 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1813 refs = btrfs_shared_data_ref_count(leaf, sref);
1814 } else {
1815 refs = 1;
1816 BUG_ON(refs_to_mod != -1);
1817 }
1818
1819 BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
1820 refs += refs_to_mod;
1821
1822 if (refs > 0) {
1823 if (type == BTRFS_EXTENT_DATA_REF_KEY)
1824 btrfs_set_extent_data_ref_count(leaf, dref, refs);
1825 else
1826 btrfs_set_shared_data_ref_count(leaf, sref, refs);
1827 } else {
Josef Bacikfcebe452014-05-13 17:30:47 -07001828 *last_ref = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001829 size = btrfs_extent_inline_ref_size(type);
1830 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1831 ptr = (unsigned long)iref;
1832 end = (unsigned long)ei + item_size;
1833 if (ptr + size < end)
1834 memmove_extent_buffer(leaf, ptr, ptr + size,
1835 end - ptr - size);
1836 item_size -= size;
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05001837 btrfs_truncate_item(fs_info, path, item_size, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001838 }
1839 btrfs_mark_buffer_dirty(leaf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001840}
1841
1842static noinline_for_stack
1843int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001844 struct btrfs_path *path,
1845 u64 bytenr, u64 num_bytes, u64 parent,
1846 u64 root_objectid, u64 owner,
1847 u64 offset, int refs_to_add,
1848 struct btrfs_delayed_extent_op *extent_op)
1849{
1850 struct btrfs_extent_inline_ref *iref;
1851 int ret;
1852
Nikolay Borisov867cc1f2018-06-20 15:48:48 +03001853 ret = lookup_inline_extent_backref(trans, path, &iref, bytenr,
1854 num_bytes, parent, root_objectid,
1855 owner, offset, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001856 if (ret == 0) {
1857 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
Nikolay Borisov61a18f12018-06-20 15:48:49 +03001858 update_inline_extent_backref(path, iref, refs_to_add,
1859 extent_op, NULL);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001860 } else if (ret == -ENOENT) {
Nikolay Borisova639cde2018-06-20 15:49:10 +03001861 setup_inline_extent_backref(trans->fs_info, path, iref, parent,
Jeff Mahoney143bede2012-03-01 14:56:26 +01001862 root_objectid, owner, offset,
1863 refs_to_add, extent_op);
1864 ret = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001865 }
1866 return ret;
1867}
1868
1869static int insert_extent_backref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001870 struct btrfs_path *path,
1871 u64 bytenr, u64 parent, u64 root_objectid,
1872 u64 owner, u64 offset, int refs_to_add)
1873{
1874 int ret;
1875 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1876 BUG_ON(refs_to_add != 1);
Nikolay Borisov10728402018-06-20 15:48:43 +03001877 ret = insert_tree_block_ref(trans, path, bytenr, parent,
1878 root_objectid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001879 } else {
Nikolay Borisov62b895a2018-06-20 15:48:44 +03001880 ret = insert_extent_data_ref(trans, path, bytenr, parent,
1881 root_objectid, owner, offset,
1882 refs_to_add);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001883 }
1884 return ret;
1885}
1886
1887static int remove_extent_backref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001888 struct btrfs_path *path,
1889 struct btrfs_extent_inline_ref *iref,
Josef Bacikfcebe452014-05-13 17:30:47 -07001890 int refs_to_drop, int is_data, int *last_ref)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001891{
Jeff Mahoney143bede2012-03-01 14:56:26 +01001892 int ret = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001893
1894 BUG_ON(!is_data && refs_to_drop != 1);
1895 if (iref) {
Nikolay Borisov61a18f12018-06-20 15:48:49 +03001896 update_inline_extent_backref(path, iref, -refs_to_drop, NULL,
1897 last_ref);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001898 } else if (is_data) {
Nikolay Borisove9f62902018-06-20 15:48:46 +03001899 ret = remove_extent_data_ref(trans, path, refs_to_drop,
Josef Bacikfcebe452014-05-13 17:30:47 -07001900 last_ref);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001901 } else {
Josef Bacikfcebe452014-05-13 17:30:47 -07001902 *last_ref = 1;
Nikolay Borisov87cc7a82018-06-20 15:49:12 +03001903 ret = btrfs_del_item(trans, trans->fs_info->extent_root, path);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001904 }
1905 return ret;
1906}
1907
Jeff Mahoneyd04c6b82015-06-15 09:41:14 -04001908static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len,
1909 u64 *discarded_bytes)
Chris Mason15916de2008-11-19 21:17:22 -05001910{
Jeff Mahoney86557862015-06-15 09:41:16 -04001911 int j, ret = 0;
1912 u64 bytes_left, end;
Jeff Mahoney4d89d372015-06-15 09:41:15 -04001913 u64 aligned_start = ALIGN(start, 1 << 9);
1914
1915 if (WARN_ON(start != aligned_start)) {
1916 len -= aligned_start - start;
1917 len = round_down(len, 1 << 9);
1918 start = aligned_start;
1919 }
Jeff Mahoneyd04c6b82015-06-15 09:41:14 -04001920
1921 *discarded_bytes = 0;
Jeff Mahoney86557862015-06-15 09:41:16 -04001922
1923 if (!len)
1924 return 0;
1925
1926 end = start + len;
1927 bytes_left = len;
1928
1929 /* Skip any superblocks on this device. */
1930 for (j = 0; j < BTRFS_SUPER_MIRROR_MAX; j++) {
1931 u64 sb_start = btrfs_sb_offset(j);
1932 u64 sb_end = sb_start + BTRFS_SUPER_INFO_SIZE;
1933 u64 size = sb_start - start;
1934
1935 if (!in_range(sb_start, start, bytes_left) &&
1936 !in_range(sb_end, start, bytes_left) &&
1937 !in_range(start, sb_start, BTRFS_SUPER_INFO_SIZE))
1938 continue;
1939
1940 /*
1941 * Superblock spans beginning of range. Adjust start and
1942 * try again.
1943 */
1944 if (sb_start <= start) {
1945 start += sb_end - start;
1946 if (start > end) {
1947 bytes_left = 0;
1948 break;
1949 }
1950 bytes_left = end - start;
1951 continue;
1952 }
1953
1954 if (size) {
1955 ret = blkdev_issue_discard(bdev, start >> 9, size >> 9,
1956 GFP_NOFS, 0);
1957 if (!ret)
1958 *discarded_bytes += size;
1959 else if (ret != -EOPNOTSUPP)
1960 return ret;
1961 }
1962
1963 start = sb_end;
1964 if (start > end) {
1965 bytes_left = 0;
1966 break;
1967 }
1968 bytes_left = end - start;
1969 }
1970
1971 if (bytes_left) {
1972 ret = blkdev_issue_discard(bdev, start >> 9, bytes_left >> 9,
Jeff Mahoney4d89d372015-06-15 09:41:15 -04001973 GFP_NOFS, 0);
1974 if (!ret)
Jeff Mahoney86557862015-06-15 09:41:16 -04001975 *discarded_bytes += bytes_left;
Jeff Mahoney4d89d372015-06-15 09:41:15 -04001976 }
Jeff Mahoneyd04c6b82015-06-15 09:41:14 -04001977 return ret;
Chris Mason15916de2008-11-19 21:17:22 -05001978}
Chris Mason15916de2008-11-19 21:17:22 -05001979
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04001980int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr,
Filipe Manana1edb647b2014-12-08 14:01:12 +00001981 u64 num_bytes, u64 *actual_bytes)
Liu Hui1f3c79a2009-01-05 15:57:51 -05001982{
Liu Hui1f3c79a2009-01-05 15:57:51 -05001983 int ret;
Li Dongyang5378e602011-03-24 10:24:27 +00001984 u64 discarded_bytes = 0;
Jan Schmidta1d3c472011-08-04 17:15:33 +02001985 struct btrfs_bio *bbio = NULL;
Liu Hui1f3c79a2009-01-05 15:57:51 -05001986
Christoph Hellwige244a0a2009-10-14 09:24:59 -04001987
Filipe Manana29992412016-05-27 17:42:05 +01001988 /*
1989 * Avoid races with device replace and make sure our bbio has devices
1990 * associated to its stripes that don't go away while we are discarding.
1991 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001992 btrfs_bio_counter_inc_blocked(fs_info);
Liu Hui1f3c79a2009-01-05 15:57:51 -05001993 /* Tell the block device(s) that the sectors can be discarded */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04001994 ret = btrfs_map_block(fs_info, BTRFS_MAP_DISCARD, bytenr, &num_bytes,
1995 &bbio, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001996 /* Error condition is -ENOMEM */
Liu Hui1f3c79a2009-01-05 15:57:51 -05001997 if (!ret) {
Jan Schmidta1d3c472011-08-04 17:15:33 +02001998 struct btrfs_bio_stripe *stripe = bbio->stripes;
Liu Hui1f3c79a2009-01-05 15:57:51 -05001999 int i;
2000
Liu Hui1f3c79a2009-01-05 15:57:51 -05002001
Jan Schmidta1d3c472011-08-04 17:15:33 +02002002 for (i = 0; i < bbio->num_stripes; i++, stripe++) {
Jeff Mahoneyd04c6b82015-06-15 09:41:14 -04002003 u64 bytes;
Anand Jain38b5f682017-11-29 18:53:43 +08002004 struct request_queue *req_q;
2005
Filipe Manana627e0872018-01-30 18:40:22 +00002006 if (!stripe->dev->bdev) {
2007 ASSERT(btrfs_test_opt(fs_info, DEGRADED));
2008 continue;
2009 }
Anand Jain38b5f682017-11-29 18:53:43 +08002010 req_q = bdev_get_queue(stripe->dev->bdev);
2011 if (!blk_queue_discard(req_q))
Josef Bacikd5e20032011-08-04 14:52:27 +00002012 continue;
2013
Li Dongyang5378e602011-03-24 10:24:27 +00002014 ret = btrfs_issue_discard(stripe->dev->bdev,
2015 stripe->physical,
Jeff Mahoneyd04c6b82015-06-15 09:41:14 -04002016 stripe->length,
2017 &bytes);
Li Dongyang5378e602011-03-24 10:24:27 +00002018 if (!ret)
Jeff Mahoneyd04c6b82015-06-15 09:41:14 -04002019 discarded_bytes += bytes;
Li Dongyang5378e602011-03-24 10:24:27 +00002020 else if (ret != -EOPNOTSUPP)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002021 break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
Josef Bacikd5e20032011-08-04 14:52:27 +00002022
2023 /*
2024 * Just in case we get back EOPNOTSUPP for some reason,
2025 * just ignore the return value so we don't screw up
2026 * people calling discard_extent.
2027 */
2028 ret = 0;
Liu Hui1f3c79a2009-01-05 15:57:51 -05002029 }
Zhao Lei6e9606d2015-01-20 15:11:34 +08002030 btrfs_put_bbio(bbio);
Liu Hui1f3c79a2009-01-05 15:57:51 -05002031 }
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002032 btrfs_bio_counter_dec(fs_info);
Li Dongyang5378e602011-03-24 10:24:27 +00002033
2034 if (actual_bytes)
2035 *actual_bytes = discarded_bytes;
2036
Liu Hui1f3c79a2009-01-05 15:57:51 -05002037
David Woodhouse53b381b2013-01-29 18:40:14 -05002038 if (ret == -EOPNOTSUPP)
2039 ret = 0;
Liu Hui1f3c79a2009-01-05 15:57:51 -05002040 return ret;
Liu Hui1f3c79a2009-01-05 15:57:51 -05002041}
2042
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002043/* Can return -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002044int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
Josef Bacik84f7d8e2017-09-29 15:43:49 -04002045 struct btrfs_root *root,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002046 u64 bytenr, u64 num_bytes, u64 parent,
Filipe Mananab06c4bf2015-10-23 07:52:54 +01002047 u64 root_objectid, u64 owner, u64 offset)
Zheng Yan31840ae2008-09-23 13:14:14 -04002048{
Josef Bacik84f7d8e2017-09-29 15:43:49 -04002049 struct btrfs_fs_info *fs_info = root->fs_info;
Qu Wenruoed4f2552019-04-04 14:45:31 +08002050 struct btrfs_ref generic_ref = { 0 };
Omar Sandovald7eae342017-06-06 16:45:31 -07002051 int old_ref_mod, new_ref_mod;
Zheng Yan31840ae2008-09-23 13:14:14 -04002052 int ret;
Arne Jansen66d7e7f2011-09-12 15:26:38 +02002053
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002054 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
2055 root_objectid == BTRFS_TREE_LOG_OBJECTID);
Zheng Yan31840ae2008-09-23 13:14:14 -04002056
Josef Bacikfd708b82017-09-29 15:43:50 -04002057 btrfs_ref_tree_mod(root, bytenr, num_bytes, parent, root_objectid,
2058 owner, offset, BTRFS_ADD_DELAYED_REF);
2059
Qu Wenruoed4f2552019-04-04 14:45:31 +08002060 btrfs_init_generic_ref(&generic_ref, BTRFS_ADD_DELAYED_REF, bytenr,
2061 num_bytes, parent);
2062 generic_ref.real_root = root->root_key.objectid;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002063 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
Qu Wenruoed4f2552019-04-04 14:45:31 +08002064 btrfs_init_tree_ref(&generic_ref, (int)owner, root_objectid);
2065 ret = btrfs_add_delayed_tree_ref(trans, &generic_ref,
2066 NULL, &old_ref_mod, &new_ref_mod);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002067 } else {
Nikolay Borisov88a979c2018-06-20 15:48:54 +03002068 ret = btrfs_add_delayed_data_ref(trans, bytenr,
Omar Sandoval7be07912017-06-06 16:45:30 -07002069 num_bytes, parent,
2070 root_objectid, owner, offset,
Omar Sandovald7eae342017-06-06 16:45:31 -07002071 0, BTRFS_ADD_DELAYED_REF,
2072 &old_ref_mod, &new_ref_mod);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002073 }
Omar Sandovald7eae342017-06-06 16:45:31 -07002074
Nikolay Borisov29d2b842018-03-30 12:58:47 +03002075 if (ret == 0 && old_ref_mod < 0 && new_ref_mod >= 0) {
2076 bool metadata = owner < BTRFS_FIRST_FREE_OBJECTID;
2077
2078 add_pinned_bytes(fs_info, -num_bytes, metadata, root_objectid);
2079 }
Omar Sandovald7eae342017-06-06 16:45:31 -07002080
Zheng Yan31840ae2008-09-23 13:14:14 -04002081 return ret;
2082}
2083
Nikolay Borisovbd3c6852018-06-18 14:59:25 +03002084/*
2085 * __btrfs_inc_extent_ref - insert backreference for a given extent
2086 *
2087 * @trans: Handle of transaction
2088 *
2089 * @node: The delayed ref node used to get the bytenr/length for
2090 * extent whose references are incremented.
2091 *
2092 * @parent: If this is a shared extent (BTRFS_SHARED_DATA_REF_KEY/
2093 * BTRFS_SHARED_BLOCK_REF_KEY) then it holds the logical
2094 * bytenr of the parent block. Since new extents are always
2095 * created with indirect references, this will only be the case
2096 * when relocating a shared extent. In that case, root_objectid
2097 * will be BTRFS_TREE_RELOC_OBJECTID. Otheriwse, parent must
2098 * be 0
2099 *
2100 * @root_objectid: The id of the root where this modification has originated,
2101 * this can be either one of the well-known metadata trees or
2102 * the subvolume id which references this extent.
2103 *
2104 * @owner: For data extents it is the inode number of the owning file.
2105 * For metadata extents this parameter holds the level in the
2106 * tree of the extent.
2107 *
2108 * @offset: For metadata extents the offset is ignored and is currently
2109 * always passed as 0. For data extents it is the fileoffset
2110 * this extent belongs to.
2111 *
2112 * @refs_to_add Number of references to add
2113 *
2114 * @extent_op Pointer to a structure, holding information necessary when
2115 * updating a tree block's flags
2116 *
2117 */
Chris Mason925baed2008-06-25 16:01:30 -04002118static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
Qu Wenruoc682f9b2015-03-17 16:59:47 +08002119 struct btrfs_delayed_ref_node *node,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002120 u64 parent, u64 root_objectid,
2121 u64 owner, u64 offset, int refs_to_add,
2122 struct btrfs_delayed_extent_op *extent_op)
Chris Mason56bec292009-03-13 10:10:06 -04002123{
Chris Mason5caf2a02007-04-02 11:20:42 -04002124 struct btrfs_path *path;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002125 struct extent_buffer *leaf;
Chris Mason234b63a2007-03-13 10:46:10 -04002126 struct btrfs_extent_item *item;
Josef Bacikfcebe452014-05-13 17:30:47 -07002127 struct btrfs_key key;
Qu Wenruoc682f9b2015-03-17 16:59:47 +08002128 u64 bytenr = node->bytenr;
2129 u64 num_bytes = node->num_bytes;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002130 u64 refs;
2131 int ret;
Chris Mason037e6392007-03-07 11:50:24 -05002132
Chris Mason5caf2a02007-04-02 11:20:42 -04002133 path = btrfs_alloc_path();
Chris Mason54aa1f42007-06-22 14:16:25 -04002134 if (!path)
2135 return -ENOMEM;
Chris Mason26b80032007-08-08 20:17:12 -04002136
David Sterbae4058b52015-11-27 16:31:35 +01002137 path->reada = READA_FORWARD;
Chris Masonb9473432009-03-13 11:00:37 -04002138 path->leave_spinning = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002139 /* this will setup the path even if it fails to insert the back ref */
Nikolay Borisova639cde2018-06-20 15:49:10 +03002140 ret = insert_inline_extent_backref(trans, path, bytenr, num_bytes,
2141 parent, root_objectid, owner,
2142 offset, refs_to_add, extent_op);
Qu Wenruo0ed47922015-04-16 16:55:08 +08002143 if ((ret < 0 && ret != -EAGAIN) || !ret)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002144 goto out;
Josef Bacikfcebe452014-05-13 17:30:47 -07002145
2146 /*
2147 * Ok we had -EAGAIN which means we didn't have space to insert and
2148 * inline extent ref, so just update the reference count and add a
2149 * normal backref.
2150 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002151 leaf = path->nodes[0];
Josef Bacikfcebe452014-05-13 17:30:47 -07002152 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002153 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2154 refs = btrfs_extent_refs(leaf, item);
2155 btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
2156 if (extent_op)
2157 __run_delayed_extent_op(extent_op, leaf, item);
Zheng Yan31840ae2008-09-23 13:14:14 -04002158
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002159 btrfs_mark_buffer_dirty(leaf);
David Sterbab3b4aa72011-04-21 01:20:15 +02002160 btrfs_release_path(path);
Chris Mason7bb86312007-12-11 09:25:06 -05002161
David Sterbae4058b52015-11-27 16:31:35 +01002162 path->reada = READA_FORWARD;
Chris Masonb9473432009-03-13 11:00:37 -04002163 path->leave_spinning = 1;
Chris Mason56bec292009-03-13 10:10:06 -04002164 /* now insert the actual backref */
Nikolay Borisov37593412018-06-20 15:48:45 +03002165 ret = insert_extent_backref(trans, path, bytenr, parent, root_objectid,
2166 owner, offset, refs_to_add);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002167 if (ret)
Jeff Mahoney66642832016-06-10 18:19:25 -04002168 btrfs_abort_transaction(trans, ret);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002169out:
Chris Mason74493f72007-12-11 09:25:06 -05002170 btrfs_free_path(path);
Liu Bo30d133f2013-10-11 16:30:23 +08002171 return ret;
Chris Mason02217ed2007-03-02 16:08:05 -05002172}
2173
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002174static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002175 struct btrfs_delayed_ref_node *node,
2176 struct btrfs_delayed_extent_op *extent_op,
2177 int insert_reserved)
Chris Masone9d0b132007-08-10 14:06:19 -04002178{
Chris Mason56bec292009-03-13 10:10:06 -04002179 int ret = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002180 struct btrfs_delayed_data_ref *ref;
2181 struct btrfs_key ins;
2182 u64 parent = 0;
2183 u64 ref_root = 0;
2184 u64 flags = 0;
Chris Mason56bec292009-03-13 10:10:06 -04002185
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002186 ins.objectid = node->bytenr;
2187 ins.offset = node->num_bytes;
2188 ins.type = BTRFS_EXTENT_ITEM_KEY;
Chris Mason56bec292009-03-13 10:10:06 -04002189
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002190 ref = btrfs_delayed_node_to_data_ref(node);
Nikolay Borisov2bf98ef2018-06-20 15:49:00 +03002191 trace_run_delayed_data_ref(trans->fs_info, node, ref, node->action);
Liu Bo599c75e2013-07-16 19:03:36 +08002192
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002193 if (node->type == BTRFS_SHARED_DATA_REF_KEY)
2194 parent = ref->parent;
Josef Bacikfcebe452014-05-13 17:30:47 -07002195 ref_root = ref->root;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002196
2197 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
Josef Bacik3173a182013-03-07 14:22:04 -05002198 if (extent_op)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002199 flags |= extent_op->flags_to_set;
Nikolay Borisovef89b822018-06-20 15:48:58 +03002200 ret = alloc_reserved_file_extent(trans, parent, ref_root,
2201 flags, ref->objectid,
2202 ref->offset, &ins,
2203 node->ref_mod);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002204 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
Nikolay Borisov2590d0f2018-06-20 15:48:59 +03002205 ret = __btrfs_inc_extent_ref(trans, node, parent, ref_root,
2206 ref->objectid, ref->offset,
2207 node->ref_mod, extent_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002208 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
Nikolay Borisove72cb922018-06-20 15:48:57 +03002209 ret = __btrfs_free_extent(trans, node, parent,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002210 ref_root, ref->objectid,
2211 ref->offset, node->ref_mod,
Qu Wenruoc682f9b2015-03-17 16:59:47 +08002212 extent_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002213 } else {
2214 BUG();
2215 }
Chris Mason56bec292009-03-13 10:10:06 -04002216 return ret;
2217}
2218
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002219static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
2220 struct extent_buffer *leaf,
2221 struct btrfs_extent_item *ei)
2222{
2223 u64 flags = btrfs_extent_flags(leaf, ei);
2224 if (extent_op->update_flags) {
2225 flags |= extent_op->flags_to_set;
2226 btrfs_set_extent_flags(leaf, ei, flags);
2227 }
2228
2229 if (extent_op->update_key) {
2230 struct btrfs_tree_block_info *bi;
2231 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
2232 bi = (struct btrfs_tree_block_info *)(ei + 1);
2233 btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
2234 }
2235}
2236
2237static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
Josef Bacikd2788502017-09-29 15:43:57 -04002238 struct btrfs_delayed_ref_head *head,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002239 struct btrfs_delayed_extent_op *extent_op)
2240{
Nikolay Borisov20b9a2d2018-06-20 15:49:01 +03002241 struct btrfs_fs_info *fs_info = trans->fs_info;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002242 struct btrfs_key key;
2243 struct btrfs_path *path;
2244 struct btrfs_extent_item *ei;
2245 struct extent_buffer *leaf;
2246 u32 item_size;
2247 int ret;
2248 int err = 0;
Josef Bacikb1c79e02013-05-09 13:49:30 -04002249 int metadata = !extent_op->is_data;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002250
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002251 if (trans->aborted)
2252 return 0;
2253
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002254 if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA))
Josef Bacik3173a182013-03-07 14:22:04 -05002255 metadata = 0;
2256
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002257 path = btrfs_alloc_path();
2258 if (!path)
2259 return -ENOMEM;
2260
Josef Bacikd2788502017-09-29 15:43:57 -04002261 key.objectid = head->bytenr;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002262
Josef Bacik3173a182013-03-07 14:22:04 -05002263 if (metadata) {
Josef Bacik3173a182013-03-07 14:22:04 -05002264 key.type = BTRFS_METADATA_ITEM_KEY;
Josef Bacikb1c79e02013-05-09 13:49:30 -04002265 key.offset = extent_op->level;
Josef Bacik3173a182013-03-07 14:22:04 -05002266 } else {
2267 key.type = BTRFS_EXTENT_ITEM_KEY;
Josef Bacikd2788502017-09-29 15:43:57 -04002268 key.offset = head->num_bytes;
Josef Bacik3173a182013-03-07 14:22:04 -05002269 }
2270
2271again:
David Sterbae4058b52015-11-27 16:31:35 +01002272 path->reada = READA_FORWARD;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002273 path->leave_spinning = 1;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002274 ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002275 if (ret < 0) {
2276 err = ret;
2277 goto out;
2278 }
2279 if (ret > 0) {
Josef Bacik3173a182013-03-07 14:22:04 -05002280 if (metadata) {
Filipe David Borba Manana55994882013-10-18 15:42:56 +01002281 if (path->slots[0] > 0) {
2282 path->slots[0]--;
2283 btrfs_item_key_to_cpu(path->nodes[0], &key,
2284 path->slots[0]);
Josef Bacikd2788502017-09-29 15:43:57 -04002285 if (key.objectid == head->bytenr &&
Filipe David Borba Manana55994882013-10-18 15:42:56 +01002286 key.type == BTRFS_EXTENT_ITEM_KEY &&
Josef Bacikd2788502017-09-29 15:43:57 -04002287 key.offset == head->num_bytes)
Filipe David Borba Manana55994882013-10-18 15:42:56 +01002288 ret = 0;
2289 }
2290 if (ret > 0) {
2291 btrfs_release_path(path);
2292 metadata = 0;
Josef Bacik3173a182013-03-07 14:22:04 -05002293
Josef Bacikd2788502017-09-29 15:43:57 -04002294 key.objectid = head->bytenr;
2295 key.offset = head->num_bytes;
Filipe David Borba Manana55994882013-10-18 15:42:56 +01002296 key.type = BTRFS_EXTENT_ITEM_KEY;
2297 goto again;
2298 }
2299 } else {
2300 err = -EIO;
2301 goto out;
Josef Bacik3173a182013-03-07 14:22:04 -05002302 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002303 }
2304
2305 leaf = path->nodes[0];
2306 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
Nikolay Borisovba3c2b12018-06-26 16:57:36 +03002307
David Sterba6d8ff4e2018-06-26 16:20:59 +02002308 if (unlikely(item_size < sizeof(*ei))) {
Nikolay Borisovba3c2b12018-06-26 16:57:36 +03002309 err = -EINVAL;
2310 btrfs_print_v0_err(fs_info);
2311 btrfs_abort_transaction(trans, err);
2312 goto out;
2313 }
2314
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002315 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2316 __run_delayed_extent_op(extent_op, leaf, ei);
2317
2318 btrfs_mark_buffer_dirty(leaf);
2319out:
2320 btrfs_free_path(path);
2321 return err;
2322}
2323
2324static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002325 struct btrfs_delayed_ref_node *node,
2326 struct btrfs_delayed_extent_op *extent_op,
2327 int insert_reserved)
2328{
2329 int ret = 0;
2330 struct btrfs_delayed_tree_ref *ref;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002331 u64 parent = 0;
2332 u64 ref_root = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002333
2334 ref = btrfs_delayed_node_to_tree_ref(node);
Nikolay Borisovf97806f2018-06-20 15:49:04 +03002335 trace_run_delayed_tree_ref(trans->fs_info, node, ref, node->action);
Liu Bo599c75e2013-07-16 19:03:36 +08002336
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002337 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2338 parent = ref->parent;
Josef Bacikfcebe452014-05-13 17:30:47 -07002339 ref_root = ref->root;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002340
Liu Bo02794222016-09-14 19:19:05 -07002341 if (node->ref_mod != 1) {
Nikolay Borisovf97806f2018-06-20 15:49:04 +03002342 btrfs_err(trans->fs_info,
Liu Bo02794222016-09-14 19:19:05 -07002343 "btree block(%llu) has %d references rather than 1: action %d ref_root %llu parent %llu",
2344 node->bytenr, node->ref_mod, node->action, ref_root,
2345 parent);
2346 return -EIO;
2347 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002348 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
Josef Bacik3173a182013-03-07 14:22:04 -05002349 BUG_ON(!extent_op || !extent_op->update_flags);
Nikolay Borisov21ebfbe2018-05-21 12:27:22 +03002350 ret = alloc_reserved_tree_block(trans, node, extent_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002351 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
Nikolay Borisov2590d0f2018-06-20 15:48:59 +03002352 ret = __btrfs_inc_extent_ref(trans, node, parent, ref_root,
2353 ref->level, 0, 1, extent_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002354 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
Nikolay Borisove72cb922018-06-20 15:48:57 +03002355 ret = __btrfs_free_extent(trans, node, parent, ref_root,
Qu Wenruoc682f9b2015-03-17 16:59:47 +08002356 ref->level, 0, 1, extent_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002357 } else {
2358 BUG();
2359 }
2360 return ret;
2361}
2362
Chris Mason56bec292009-03-13 10:10:06 -04002363/* helper function to actually process a single delayed ref entry */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002364static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002365 struct btrfs_delayed_ref_node *node,
2366 struct btrfs_delayed_extent_op *extent_op,
2367 int insert_reserved)
Chris Mason56bec292009-03-13 10:10:06 -04002368{
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002369 int ret = 0;
2370
Josef Bacik857cc2f2013-10-07 15:21:08 -04002371 if (trans->aborted) {
2372 if (insert_reserved)
Nikolay Borisov5fac7f92018-06-20 15:49:11 +03002373 btrfs_pin_extent(trans->fs_info, node->bytenr,
Josef Bacik857cc2f2013-10-07 15:21:08 -04002374 node->num_bytes, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002375 return 0;
Josef Bacik857cc2f2013-10-07 15:21:08 -04002376 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002377
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002378 if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
2379 node->type == BTRFS_SHARED_BLOCK_REF_KEY)
Nikolay Borisovf97806f2018-06-20 15:49:04 +03002380 ret = run_delayed_tree_ref(trans, node, extent_op,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002381 insert_reserved);
2382 else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
2383 node->type == BTRFS_SHARED_DATA_REF_KEY)
Nikolay Borisov2bf98ef2018-06-20 15:49:00 +03002384 ret = run_delayed_data_ref(trans, node, extent_op,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002385 insert_reserved);
2386 else
2387 BUG();
Josef Bacik80ee54b2018-10-11 15:54:22 -04002388 if (ret && insert_reserved)
2389 btrfs_pin_extent(trans->fs_info, node->bytenr,
2390 node->num_bytes, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002391 return ret;
Chris Masone9d0b132007-08-10 14:06:19 -04002392}
2393
Qu Wenruoc6fc2452015-03-30 17:03:00 +08002394static inline struct btrfs_delayed_ref_node *
Chris Mason56bec292009-03-13 10:10:06 -04002395select_delayed_ref(struct btrfs_delayed_ref_head *head)
Chris Masona28ec192007-03-06 20:08:01 -05002396{
Filipe Mananacffc3372015-07-09 13:13:44 +01002397 struct btrfs_delayed_ref_node *ref;
2398
Liu Boe3d03962018-08-23 03:51:50 +08002399 if (RB_EMPTY_ROOT(&head->ref_tree.rb_root))
Qu Wenruoc6fc2452015-03-30 17:03:00 +08002400 return NULL;
Josef Bacikd7df2c72014-01-23 09:21:38 -05002401
Filipe Mananacffc3372015-07-09 13:13:44 +01002402 /*
2403 * Select a delayed ref of type BTRFS_ADD_DELAYED_REF first.
2404 * This is to prevent a ref count from going down to zero, which deletes
2405 * the extent item from the extent tree, when there still are references
2406 * to add, which would fail because they would not find the extent item.
2407 */
Wang Xiaoguang1d57ee92016-10-26 18:07:33 +08002408 if (!list_empty(&head->ref_add_list))
2409 return list_first_entry(&head->ref_add_list,
2410 struct btrfs_delayed_ref_node, add_list);
Filipe Mananacffc3372015-07-09 13:13:44 +01002411
Liu Boe3d03962018-08-23 03:51:50 +08002412 ref = rb_entry(rb_first_cached(&head->ref_tree),
Josef Bacik0e0adbc2017-10-19 14:16:00 -04002413 struct btrfs_delayed_ref_node, ref_node);
Wang Xiaoguang1d57ee92016-10-26 18:07:33 +08002414 ASSERT(list_empty(&ref->add_list));
2415 return ref;
Chris Mason56bec292009-03-13 10:10:06 -04002416}
2417
Josef Bacik2eadaa22017-09-29 15:43:52 -04002418static void unselect_delayed_ref_head(struct btrfs_delayed_ref_root *delayed_refs,
2419 struct btrfs_delayed_ref_head *head)
2420{
2421 spin_lock(&delayed_refs->lock);
2422 head->processing = 0;
2423 delayed_refs->num_heads_ready++;
2424 spin_unlock(&delayed_refs->lock);
2425 btrfs_delayed_ref_unlock(head);
2426}
2427
Josef Bacikbedc66172018-12-03 10:20:31 -05002428static struct btrfs_delayed_extent_op *cleanup_extent_op(
2429 struct btrfs_delayed_ref_head *head)
Josef Bacikb00e6252017-09-29 15:43:53 -04002430{
2431 struct btrfs_delayed_extent_op *extent_op = head->extent_op;
Josef Bacikbedc66172018-12-03 10:20:31 -05002432
2433 if (!extent_op)
2434 return NULL;
2435
2436 if (head->must_insert_reserved) {
2437 head->extent_op = NULL;
2438 btrfs_free_delayed_extent_op(extent_op);
2439 return NULL;
2440 }
2441 return extent_op;
2442}
2443
2444static int run_and_cleanup_extent_op(struct btrfs_trans_handle *trans,
2445 struct btrfs_delayed_ref_head *head)
2446{
2447 struct btrfs_delayed_extent_op *extent_op;
Josef Bacikb00e6252017-09-29 15:43:53 -04002448 int ret;
2449
Josef Bacikbedc66172018-12-03 10:20:31 -05002450 extent_op = cleanup_extent_op(head);
Josef Bacikb00e6252017-09-29 15:43:53 -04002451 if (!extent_op)
2452 return 0;
2453 head->extent_op = NULL;
Josef Bacikb00e6252017-09-29 15:43:53 -04002454 spin_unlock(&head->lock);
Nikolay Borisov20b9a2d2018-06-20 15:49:01 +03002455 ret = run_delayed_extent_op(trans, head, extent_op);
Josef Bacikb00e6252017-09-29 15:43:53 -04002456 btrfs_free_delayed_extent_op(extent_op);
2457 return ret ? ret : 1;
2458}
2459
Josef Bacik31890da2018-11-21 14:05:41 -05002460void btrfs_cleanup_ref_head_accounting(struct btrfs_fs_info *fs_info,
2461 struct btrfs_delayed_ref_root *delayed_refs,
2462 struct btrfs_delayed_ref_head *head)
Josef Bacik07c47772018-12-03 10:20:30 -05002463{
Josef Bacikba2c4d42018-12-03 10:20:33 -05002464 int nr_items = 1; /* Dropping this ref head update. */
Josef Bacik07c47772018-12-03 10:20:30 -05002465
2466 if (head->total_ref_mod < 0) {
2467 struct btrfs_space_info *space_info;
2468 u64 flags;
2469
2470 if (head->is_data)
2471 flags = BTRFS_BLOCK_GROUP_DATA;
2472 else if (head->is_system)
2473 flags = BTRFS_BLOCK_GROUP_SYSTEM;
2474 else
2475 flags = BTRFS_BLOCK_GROUP_METADATA;
2476 space_info = __find_space_info(fs_info, flags);
2477 ASSERT(space_info);
2478 percpu_counter_add_batch(&space_info->total_bytes_pinned,
2479 -head->num_bytes,
2480 BTRFS_TOTAL_BYTES_PINNED_BATCH);
2481
Josef Bacikba2c4d42018-12-03 10:20:33 -05002482 /*
2483 * We had csum deletions accounted for in our delayed refs rsv,
2484 * we need to drop the csum leaves for this update from our
2485 * delayed_refs_rsv.
2486 */
Josef Bacik07c47772018-12-03 10:20:30 -05002487 if (head->is_data) {
2488 spin_lock(&delayed_refs->lock);
2489 delayed_refs->pending_csums -= head->num_bytes;
2490 spin_unlock(&delayed_refs->lock);
Josef Bacikba2c4d42018-12-03 10:20:33 -05002491 nr_items += btrfs_csum_bytes_to_leaves(fs_info,
2492 head->num_bytes);
Josef Bacik07c47772018-12-03 10:20:30 -05002493 }
2494 }
2495
Josef Bacikba2c4d42018-12-03 10:20:33 -05002496 btrfs_delayed_refs_rsv_release(fs_info, nr_items);
Josef Bacik07c47772018-12-03 10:20:30 -05002497}
2498
Josef Bacik194ab0b2017-09-29 15:43:54 -04002499static int cleanup_ref_head(struct btrfs_trans_handle *trans,
Josef Bacik194ab0b2017-09-29 15:43:54 -04002500 struct btrfs_delayed_ref_head *head)
2501{
Nikolay Borisovf9871ed2018-06-20 15:49:03 +03002502
2503 struct btrfs_fs_info *fs_info = trans->fs_info;
Josef Bacik194ab0b2017-09-29 15:43:54 -04002504 struct btrfs_delayed_ref_root *delayed_refs;
2505 int ret;
2506
2507 delayed_refs = &trans->transaction->delayed_refs;
2508
Josef Bacikbedc66172018-12-03 10:20:31 -05002509 ret = run_and_cleanup_extent_op(trans, head);
Josef Bacik194ab0b2017-09-29 15:43:54 -04002510 if (ret < 0) {
2511 unselect_delayed_ref_head(delayed_refs, head);
2512 btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret);
2513 return ret;
2514 } else if (ret) {
2515 return ret;
2516 }
2517
2518 /*
2519 * Need to drop our head ref lock and re-acquire the delayed ref lock
2520 * and then re-check to make sure nobody got added.
2521 */
2522 spin_unlock(&head->lock);
2523 spin_lock(&delayed_refs->lock);
2524 spin_lock(&head->lock);
Liu Boe3d03962018-08-23 03:51:50 +08002525 if (!RB_EMPTY_ROOT(&head->ref_tree.rb_root) || head->extent_op) {
Josef Bacik194ab0b2017-09-29 15:43:54 -04002526 spin_unlock(&head->lock);
2527 spin_unlock(&delayed_refs->lock);
2528 return 1;
2529 }
Josef Bacikd7baffd2018-12-03 10:20:29 -05002530 btrfs_delete_ref_head(delayed_refs, head);
Josef Bacikc1103f72017-09-29 15:43:56 -04002531 spin_unlock(&head->lock);
Nikolay Borisov1e7a1422018-04-11 11:21:18 +03002532 spin_unlock(&delayed_refs->lock);
Josef Bacikc1103f72017-09-29 15:43:56 -04002533
Josef Bacikc1103f72017-09-29 15:43:56 -04002534 if (head->must_insert_reserved) {
Josef Bacikd2788502017-09-29 15:43:57 -04002535 btrfs_pin_extent(fs_info, head->bytenr,
2536 head->num_bytes, 1);
Josef Bacikc1103f72017-09-29 15:43:56 -04002537 if (head->is_data) {
Josef Bacikd2788502017-09-29 15:43:57 -04002538 ret = btrfs_del_csums(trans, fs_info, head->bytenr,
2539 head->num_bytes);
Josef Bacikc1103f72017-09-29 15:43:56 -04002540 }
2541 }
2542
Josef Bacik31890da2018-11-21 14:05:41 -05002543 btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head);
Josef Bacik07c47772018-12-03 10:20:30 -05002544
2545 trace_run_delayed_ref_head(fs_info, head, 0);
Josef Bacikc1103f72017-09-29 15:43:56 -04002546 btrfs_delayed_ref_unlock(head);
Josef Bacikd2788502017-09-29 15:43:57 -04002547 btrfs_put_delayed_ref_head(head);
Josef Bacik194ab0b2017-09-29 15:43:54 -04002548 return 0;
2549}
2550
Nikolay Borisovb1cdbcb2018-08-15 10:39:54 +03002551static struct btrfs_delayed_ref_head *btrfs_obtain_ref_head(
2552 struct btrfs_trans_handle *trans)
2553{
2554 struct btrfs_delayed_ref_root *delayed_refs =
2555 &trans->transaction->delayed_refs;
2556 struct btrfs_delayed_ref_head *head = NULL;
2557 int ret;
2558
2559 spin_lock(&delayed_refs->lock);
Lu Fengqi5637c742018-10-11 13:40:33 +08002560 head = btrfs_select_ref_head(delayed_refs);
Nikolay Borisovb1cdbcb2018-08-15 10:39:54 +03002561 if (!head) {
2562 spin_unlock(&delayed_refs->lock);
2563 return head;
2564 }
2565
2566 /*
2567 * Grab the lock that says we are going to process all the refs for
2568 * this head
2569 */
Lu Fengqi9e920a62018-10-11 13:40:34 +08002570 ret = btrfs_delayed_ref_lock(delayed_refs, head);
Nikolay Borisovb1cdbcb2018-08-15 10:39:54 +03002571 spin_unlock(&delayed_refs->lock);
2572
2573 /*
2574 * We may have dropped the spin lock to get the head mutex lock, and
2575 * that might have given someone else time to free the head. If that's
2576 * true, it has been removed from our list and we can move on.
2577 */
2578 if (ret == -EAGAIN)
2579 head = ERR_PTR(-EAGAIN);
2580
2581 return head;
2582}
2583
Nikolay Borisove7261382018-08-15 10:39:55 +03002584static int btrfs_run_delayed_refs_for_head(struct btrfs_trans_handle *trans,
2585 struct btrfs_delayed_ref_head *locked_ref,
2586 unsigned long *run_refs)
2587{
2588 struct btrfs_fs_info *fs_info = trans->fs_info;
2589 struct btrfs_delayed_ref_root *delayed_refs;
2590 struct btrfs_delayed_extent_op *extent_op;
2591 struct btrfs_delayed_ref_node *ref;
2592 int must_insert_reserved = 0;
2593 int ret;
2594
2595 delayed_refs = &trans->transaction->delayed_refs;
2596
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002597 lockdep_assert_held(&locked_ref->mutex);
2598 lockdep_assert_held(&locked_ref->lock);
2599
Nikolay Borisove7261382018-08-15 10:39:55 +03002600 while ((ref = select_delayed_ref(locked_ref))) {
2601 if (ref->seq &&
2602 btrfs_check_delayed_seq(fs_info, ref->seq)) {
2603 spin_unlock(&locked_ref->lock);
2604 unselect_delayed_ref_head(delayed_refs, locked_ref);
2605 return -EAGAIN;
2606 }
2607
2608 (*run_refs)++;
2609 ref->in_tree = 0;
2610 rb_erase_cached(&ref->ref_node, &locked_ref->ref_tree);
2611 RB_CLEAR_NODE(&ref->ref_node);
2612 if (!list_empty(&ref->add_list))
2613 list_del(&ref->add_list);
2614 /*
2615 * When we play the delayed ref, also correct the ref_mod on
2616 * head
2617 */
2618 switch (ref->action) {
2619 case BTRFS_ADD_DELAYED_REF:
2620 case BTRFS_ADD_DELAYED_EXTENT:
2621 locked_ref->ref_mod -= ref->ref_mod;
2622 break;
2623 case BTRFS_DROP_DELAYED_REF:
2624 locked_ref->ref_mod += ref->ref_mod;
2625 break;
2626 default:
2627 WARN_ON(1);
2628 }
2629 atomic_dec(&delayed_refs->num_entries);
2630
2631 /*
2632 * Record the must_insert_reserved flag before we drop the
2633 * spin lock.
2634 */
2635 must_insert_reserved = locked_ref->must_insert_reserved;
2636 locked_ref->must_insert_reserved = 0;
2637
2638 extent_op = locked_ref->extent_op;
2639 locked_ref->extent_op = NULL;
2640 spin_unlock(&locked_ref->lock);
2641
2642 ret = run_one_delayed_ref(trans, ref, extent_op,
2643 must_insert_reserved);
2644
2645 btrfs_free_delayed_extent_op(extent_op);
2646 if (ret) {
2647 unselect_delayed_ref_head(delayed_refs, locked_ref);
2648 btrfs_put_delayed_ref(ref);
2649 btrfs_debug(fs_info, "run_one_delayed_ref returned %d",
2650 ret);
2651 return ret;
2652 }
2653
2654 btrfs_put_delayed_ref(ref);
2655 cond_resched();
2656
2657 spin_lock(&locked_ref->lock);
2658 btrfs_merge_delayed_refs(trans, delayed_refs, locked_ref);
2659 }
2660
2661 return 0;
2662}
2663
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002664/*
2665 * Returns 0 on success or if called with an already aborted transaction.
2666 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
2667 */
Josef Bacikd7df2c72014-01-23 09:21:38 -05002668static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
Josef Bacikd7df2c72014-01-23 09:21:38 -05002669 unsigned long nr)
Chris Mason56bec292009-03-13 10:10:06 -04002670{
Nikolay Borisov0a1e4582018-03-15 16:00:27 +02002671 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Mason56bec292009-03-13 10:10:06 -04002672 struct btrfs_delayed_ref_root *delayed_refs;
Chris Mason56bec292009-03-13 10:10:06 -04002673 struct btrfs_delayed_ref_head *locked_ref = NULL;
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002674 ktime_t start = ktime_get();
Chris Mason56bec292009-03-13 10:10:06 -04002675 int ret;
Josef Bacikd7df2c72014-01-23 09:21:38 -05002676 unsigned long count = 0;
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002677 unsigned long actual_count = 0;
Chris Mason56bec292009-03-13 10:10:06 -04002678
2679 delayed_refs = &trans->transaction->delayed_refs;
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002680 do {
Chris Mason56bec292009-03-13 10:10:06 -04002681 if (!locked_ref) {
Nikolay Borisovb1cdbcb2018-08-15 10:39:54 +03002682 locked_ref = btrfs_obtain_ref_head(trans);
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002683 if (IS_ERR_OR_NULL(locked_ref)) {
2684 if (PTR_ERR(locked_ref) == -EAGAIN) {
2685 continue;
2686 } else {
2687 break;
2688 }
Chris Mason56bec292009-03-13 10:10:06 -04002689 }
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002690 count++;
Chris Mason56bec292009-03-13 10:10:06 -04002691 }
Filipe Manana2c3cf7d2015-10-22 09:47:34 +01002692 /*
2693 * We need to try and merge add/drops of the same ref since we
2694 * can run into issues with relocate dropping the implicit ref
2695 * and then it being added back again before the drop can
2696 * finish. If we merged anything we need to re-loop so we can
2697 * get a good ref.
2698 * Or we can get node references of the same type that weren't
2699 * merged when created due to bumps in the tree mod seq, and
2700 * we need to merge them to prevent adding an inline extent
2701 * backref before dropping it (triggering a BUG_ON at
2702 * insert_inline_extent_backref()).
2703 */
Josef Bacikd7df2c72014-01-23 09:21:38 -05002704 spin_lock(&locked_ref->lock);
Nikolay Borisovbe97f132018-04-19 11:06:39 +03002705 btrfs_merge_delayed_refs(trans, delayed_refs, locked_ref);
Josef Bacikae1e2062012-08-07 16:00:32 -04002706
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002707 ret = btrfs_run_delayed_refs_for_head(trans, locked_ref,
2708 &actual_count);
2709 if (ret < 0 && ret != -EAGAIN) {
2710 /*
2711 * Error, btrfs_run_delayed_refs_for_head already
2712 * unlocked everything so just bail out
2713 */
2714 return ret;
2715 } else if (!ret) {
2716 /*
2717 * Success, perform the usual cleanup of a processed
2718 * head
2719 */
Nikolay Borisovf9871ed2018-06-20 15:49:03 +03002720 ret = cleanup_ref_head(trans, locked_ref);
Josef Bacik194ab0b2017-09-29 15:43:54 -04002721 if (ret > 0 ) {
Josef Bacikb00e6252017-09-29 15:43:53 -04002722 /* We dropped our lock, we need to loop. */
2723 ret = 0;
Josef Bacikd7df2c72014-01-23 09:21:38 -05002724 continue;
Josef Bacik194ab0b2017-09-29 15:43:54 -04002725 } else if (ret) {
2726 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002727 }
Josef Bacikc1103f72017-09-29 15:43:56 -04002728 }
Chris Mason56bec292009-03-13 10:10:06 -04002729
Josef Bacikb00e6252017-09-29 15:43:53 -04002730 /*
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002731 * Either success case or btrfs_run_delayed_refs_for_head
2732 * returned -EAGAIN, meaning we need to select another head
Josef Bacikb00e6252017-09-29 15:43:53 -04002733 */
Josef Bacikb00e6252017-09-29 15:43:53 -04002734
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002735 locked_ref = NULL;
Chris Mason1887be62009-03-13 10:11:24 -04002736 cond_resched();
Nikolay Borisov0110a4c2018-08-15 10:39:56 +03002737 } while ((nr != -1 && count < nr) || locked_ref);
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002738
2739 /*
2740 * We don't want to include ref heads since we can have empty ref heads
2741 * and those will drastically skew our runtime down since we just do
2742 * accounting, no actual extent tree updates.
2743 */
2744 if (actual_count > 0) {
2745 u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start));
2746 u64 avg;
2747
2748 /*
2749 * We weigh the current average higher than our current runtime
2750 * to avoid large swings in the average.
2751 */
2752 spin_lock(&delayed_refs->lock);
2753 avg = fs_info->avg_delayed_ref_runtime * 3 + runtime;
David Sterbaf8c269d2015-01-16 17:21:12 +01002754 fs_info->avg_delayed_ref_runtime = avg >> 2; /* div by 4 */
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002755 spin_unlock(&delayed_refs->lock);
2756 }
Josef Bacikd7df2c72014-01-23 09:21:38 -05002757 return 0;
Chris Masonc3e69d52009-03-13 10:17:05 -04002758}
2759
Arne Jansen709c0482011-09-12 12:22:57 +02002760#ifdef SCRAMBLE_DELAYED_REFS
2761/*
2762 * Normally delayed refs get processed in ascending bytenr order. This
2763 * correlates in most cases to the order added. To expose dependencies on this
2764 * order, we start to process the tree in the middle instead of the beginning
2765 */
2766static u64 find_middle(struct rb_root *root)
2767{
2768 struct rb_node *n = root->rb_node;
2769 struct btrfs_delayed_ref_node *entry;
2770 int alt = 1;
2771 u64 middle;
2772 u64 first = 0, last = 0;
2773
2774 n = rb_first(root);
2775 if (n) {
2776 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2777 first = entry->bytenr;
2778 }
2779 n = rb_last(root);
2780 if (n) {
2781 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2782 last = entry->bytenr;
2783 }
2784 n = root->rb_node;
2785
2786 while (n) {
2787 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2788 WARN_ON(!entry->in_tree);
2789
2790 middle = entry->bytenr;
2791
2792 if (alt)
2793 n = n->rb_left;
2794 else
2795 n = n->rb_right;
2796
2797 alt = 1 - alt;
2798 }
2799 return middle;
2800}
2801#endif
2802
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002803static inline u64 heads_to_leaves(struct btrfs_fs_info *fs_info, u64 heads)
Josef Bacik1be41b72013-06-12 13:56:06 -04002804{
2805 u64 num_bytes;
2806
2807 num_bytes = heads * (sizeof(struct btrfs_extent_item) +
2808 sizeof(struct btrfs_extent_inline_ref));
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002809 if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA))
Josef Bacik1be41b72013-06-12 13:56:06 -04002810 num_bytes += heads * sizeof(struct btrfs_tree_block_info);
2811
2812 /*
2813 * We don't ever fill up leaves all the way so multiply by 2 just to be
Nicholas D Steeves01327612016-05-19 21:18:45 -04002814 * closer to what we're really going to want to use.
Josef Bacik1be41b72013-06-12 13:56:06 -04002815 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002816 return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(fs_info));
Josef Bacik1be41b72013-06-12 13:56:06 -04002817}
2818
Josef Bacik12621332015-02-03 07:50:16 -08002819/*
2820 * Takes the number of bytes to be csumm'ed and figures out how many leaves it
2821 * would require to store the csums for that many bytes.
2822 */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002823u64 btrfs_csum_bytes_to_leaves(struct btrfs_fs_info *fs_info, u64 csum_bytes)
Josef Bacik12621332015-02-03 07:50:16 -08002824{
2825 u64 csum_size;
2826 u64 num_csums_per_leaf;
2827 u64 num_csums;
2828
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002829 csum_size = BTRFS_MAX_ITEM_SIZE(fs_info);
Josef Bacik12621332015-02-03 07:50:16 -08002830 num_csums_per_leaf = div64_u64(csum_size,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002831 (u64)btrfs_super_csum_size(fs_info->super_copy));
2832 num_csums = div64_u64(csum_bytes, fs_info->sectorsize);
Josef Bacik12621332015-02-03 07:50:16 -08002833 num_csums += num_csums_per_leaf - 1;
2834 num_csums = div64_u64(num_csums, num_csums_per_leaf);
2835 return num_csums;
2836}
2837
Josef Bacik64403612018-12-03 10:20:36 -05002838bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info)
Josef Bacik1be41b72013-06-12 13:56:06 -04002839{
Josef Bacik64403612018-12-03 10:20:36 -05002840 struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv;
2841 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
2842 bool ret = false;
2843 u64 reserved;
Josef Bacik1be41b72013-06-12 13:56:06 -04002844
2845 spin_lock(&global_rsv->lock);
Josef Bacik64403612018-12-03 10:20:36 -05002846 reserved = global_rsv->reserved;
Josef Bacik1be41b72013-06-12 13:56:06 -04002847 spin_unlock(&global_rsv->lock);
Josef Bacik64403612018-12-03 10:20:36 -05002848
2849 /*
2850 * Since the global reserve is just kind of magic we don't really want
2851 * to rely on it to save our bacon, so if our size is more than the
2852 * delayed_refs_rsv and the global rsv then it's time to think about
2853 * bailing.
2854 */
2855 spin_lock(&delayed_refs_rsv->lock);
2856 reserved += delayed_refs_rsv->reserved;
2857 if (delayed_refs_rsv->size >= reserved)
2858 ret = true;
2859 spin_unlock(&delayed_refs_rsv->lock);
Josef Bacik1be41b72013-06-12 13:56:06 -04002860 return ret;
2861}
2862
Lu Fengqi7c861622018-10-11 13:40:36 +08002863int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans)
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002864{
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002865 u64 num_entries =
2866 atomic_read(&trans->transaction->delayed_refs.num_entries);
2867 u64 avg_runtime;
Chris Masona79b7d42014-05-22 16:18:52 -07002868 u64 val;
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002869
2870 smp_mb();
Lu Fengqi7c861622018-10-11 13:40:36 +08002871 avg_runtime = trans->fs_info->avg_delayed_ref_runtime;
Chris Masona79b7d42014-05-22 16:18:52 -07002872 val = num_entries * avg_runtime;
Wang Xiaoguangdc1a90c2016-10-26 15:23:01 +08002873 if (val >= NSEC_PER_SEC)
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002874 return 1;
Chris Masona79b7d42014-05-22 16:18:52 -07002875 if (val >= NSEC_PER_SEC / 2)
2876 return 2;
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002877
Josef Bacik64403612018-12-03 10:20:36 -05002878 return btrfs_check_space_for_delayed_refs(trans->fs_info);
Josef Bacik0a2b2a82014-01-23 10:54:11 -05002879}
2880
Chris Masonc3e69d52009-03-13 10:17:05 -04002881/*
2882 * this starts processing the delayed reference count updates and
2883 * extent insertions we have queued up so far. count can be
2884 * 0, which means to process everything in the tree at the start
2885 * of the run (but not newly added entries), or it can be some target
2886 * number you'd like to process.
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002887 *
2888 * Returns 0 on success or if called with an aborted transaction
2889 * Returns <0 on error and aborts the transaction
Chris Masonc3e69d52009-03-13 10:17:05 -04002890 */
2891int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
Nikolay Borisovc79a70b2018-03-15 17:27:37 +02002892 unsigned long count)
Chris Masonc3e69d52009-03-13 10:17:05 -04002893{
Nikolay Borisovc79a70b2018-03-15 17:27:37 +02002894 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Masonc3e69d52009-03-13 10:17:05 -04002895 struct rb_node *node;
2896 struct btrfs_delayed_ref_root *delayed_refs;
Liu Boc46effa2013-10-14 12:59:45 +08002897 struct btrfs_delayed_ref_head *head;
Chris Masonc3e69d52009-03-13 10:17:05 -04002898 int ret;
2899 int run_all = count == (unsigned long)-1;
Chris Masonc3e69d52009-03-13 10:17:05 -04002900
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002901 /* We'll clean this up in btrfs_cleanup_transaction */
2902 if (trans->aborted)
2903 return 0;
2904
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002905 if (test_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags))
Chris Mason511711a2015-12-30 07:52:35 -08002906 return 0;
2907
Chris Masonc3e69d52009-03-13 10:17:05 -04002908 delayed_refs = &trans->transaction->delayed_refs;
Liu Bo26455d32014-12-17 16:14:09 +08002909 if (count == 0)
Josef Bacikd7df2c72014-01-23 09:21:38 -05002910 count = atomic_read(&delayed_refs->num_entries) * 2;
Chris Masonbb721702013-01-29 18:44:12 -05002911
Chris Masonc3e69d52009-03-13 10:17:05 -04002912again:
Arne Jansen709c0482011-09-12 12:22:57 +02002913#ifdef SCRAMBLE_DELAYED_REFS
2914 delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
2915#endif
Nikolay Borisov0a1e4582018-03-15 16:00:27 +02002916 ret = __btrfs_run_delayed_refs(trans, count);
Josef Bacikd7df2c72014-01-23 09:21:38 -05002917 if (ret < 0) {
Jeff Mahoney66642832016-06-10 18:19:25 -04002918 btrfs_abort_transaction(trans, ret);
Josef Bacikd7df2c72014-01-23 09:21:38 -05002919 return ret;
Chris Masonc3e69d52009-03-13 10:17:05 -04002920 }
2921
Chris Mason56bec292009-03-13 10:10:06 -04002922 if (run_all) {
Josef Bacik119e80d2018-11-21 14:05:42 -05002923 btrfs_create_pending_block_groups(trans);
Josef Bacikea658ba2012-09-11 16:57:25 -04002924
Josef Bacikd7df2c72014-01-23 09:21:38 -05002925 spin_lock(&delayed_refs->lock);
Liu Bo5c9d0282018-08-23 03:51:49 +08002926 node = rb_first_cached(&delayed_refs->href_root);
Josef Bacikd7df2c72014-01-23 09:21:38 -05002927 if (!node) {
2928 spin_unlock(&delayed_refs->lock);
Chris Mason56bec292009-03-13 10:10:06 -04002929 goto out;
Josef Bacikd7df2c72014-01-23 09:21:38 -05002930 }
Josef Bacikd2788502017-09-29 15:43:57 -04002931 head = rb_entry(node, struct btrfs_delayed_ref_head,
2932 href_node);
2933 refcount_inc(&head->refs);
Chris Mason56bec292009-03-13 10:10:06 -04002934 spin_unlock(&delayed_refs->lock);
Josef Bacikd2788502017-09-29 15:43:57 -04002935
2936 /* Mutex was contended, block until it's released and retry. */
2937 mutex_lock(&head->mutex);
2938 mutex_unlock(&head->mutex);
2939
2940 btrfs_put_delayed_ref_head(head);
Josef Bacikd7df2c72014-01-23 09:21:38 -05002941 cond_resched();
Chris Mason56bec292009-03-13 10:10:06 -04002942 goto again;
2943 }
Chris Mason54aa1f42007-06-22 14:16:25 -04002944out:
Chris Masona28ec192007-03-06 20:08:01 -05002945 return 0;
2946}
2947
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002948int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04002949 struct btrfs_fs_info *fs_info,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002950 u64 bytenr, u64 num_bytes, u64 flags,
Josef Bacikb1c79e02013-05-09 13:49:30 -04002951 int level, int is_data)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002952{
2953 struct btrfs_delayed_extent_op *extent_op;
2954 int ret;
2955
Miao Xie78a61842012-11-21 02:21:28 +00002956 extent_op = btrfs_alloc_delayed_extent_op();
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002957 if (!extent_op)
2958 return -ENOMEM;
2959
2960 extent_op->flags_to_set = flags;
David Sterba35b3ad52015-11-30 16:51:29 +01002961 extent_op->update_flags = true;
2962 extent_op->update_key = false;
2963 extent_op->is_data = is_data ? true : false;
Josef Bacikb1c79e02013-05-09 13:49:30 -04002964 extent_op->level = level;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002965
Jeff Mahoney0b246af2016-06-22 18:54:23 -04002966 ret = btrfs_add_delayed_extent_op(fs_info, trans, bytenr,
Arne Jansen66d7e7f2011-09-12 15:26:38 +02002967 num_bytes, extent_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002968 if (ret)
Miao Xie78a61842012-11-21 02:21:28 +00002969 btrfs_free_delayed_extent_op(extent_op);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002970 return ret;
2971}
2972
Liu Boe4c3b2d2017-01-30 12:25:28 -08002973static noinline int check_delayed_ref(struct btrfs_root *root,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002974 struct btrfs_path *path,
2975 u64 objectid, u64 offset, u64 bytenr)
2976{
2977 struct btrfs_delayed_ref_head *head;
2978 struct btrfs_delayed_ref_node *ref;
2979 struct btrfs_delayed_data_ref *data_ref;
2980 struct btrfs_delayed_ref_root *delayed_refs;
Liu Boe4c3b2d2017-01-30 12:25:28 -08002981 struct btrfs_transaction *cur_trans;
Josef Bacik0e0adbc2017-10-19 14:16:00 -04002982 struct rb_node *node;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002983 int ret = 0;
2984
ethanwu998ac6d2018-04-29 15:59:42 +08002985 spin_lock(&root->fs_info->trans_lock);
Liu Boe4c3b2d2017-01-30 12:25:28 -08002986 cur_trans = root->fs_info->running_transaction;
ethanwu998ac6d2018-04-29 15:59:42 +08002987 if (cur_trans)
2988 refcount_inc(&cur_trans->use_count);
2989 spin_unlock(&root->fs_info->trans_lock);
Liu Boe4c3b2d2017-01-30 12:25:28 -08002990 if (!cur_trans)
2991 return 0;
2992
2993 delayed_refs = &cur_trans->delayed_refs;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002994 spin_lock(&delayed_refs->lock);
Liu Bof72ad18e2017-01-30 12:24:37 -08002995 head = btrfs_find_delayed_ref_head(delayed_refs, bytenr);
Josef Bacikd7df2c72014-01-23 09:21:38 -05002996 if (!head) {
2997 spin_unlock(&delayed_refs->lock);
ethanwu998ac6d2018-04-29 15:59:42 +08002998 btrfs_put_transaction(cur_trans);
Josef Bacikd7df2c72014-01-23 09:21:38 -05002999 return 0;
3000 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003001
3002 if (!mutex_trylock(&head->mutex)) {
Josef Bacikd2788502017-09-29 15:43:57 -04003003 refcount_inc(&head->refs);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003004 spin_unlock(&delayed_refs->lock);
3005
David Sterbab3b4aa72011-04-21 01:20:15 +02003006 btrfs_release_path(path);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003007
David Sterba8cc33e52011-05-02 15:29:25 +02003008 /*
3009 * Mutex was contended, block until it's released and let
3010 * caller try again
3011 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003012 mutex_lock(&head->mutex);
3013 mutex_unlock(&head->mutex);
Josef Bacikd2788502017-09-29 15:43:57 -04003014 btrfs_put_delayed_ref_head(head);
ethanwu998ac6d2018-04-29 15:59:42 +08003015 btrfs_put_transaction(cur_trans);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003016 return -EAGAIN;
3017 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003018 spin_unlock(&delayed_refs->lock);
Josef Bacikd7df2c72014-01-23 09:21:38 -05003019
3020 spin_lock(&head->lock);
Josef Bacik0e0adbc2017-10-19 14:16:00 -04003021 /*
3022 * XXX: We should replace this with a proper search function in the
3023 * future.
3024 */
Liu Boe3d03962018-08-23 03:51:50 +08003025 for (node = rb_first_cached(&head->ref_tree); node;
3026 node = rb_next(node)) {
Josef Bacik0e0adbc2017-10-19 14:16:00 -04003027 ref = rb_entry(node, struct btrfs_delayed_ref_node, ref_node);
Josef Bacikd7df2c72014-01-23 09:21:38 -05003028 /* If it's a shared ref we know a cross reference exists */
3029 if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) {
3030 ret = 1;
3031 break;
3032 }
3033
3034 data_ref = btrfs_delayed_node_to_data_ref(ref);
3035
3036 /*
3037 * If our ref doesn't match the one we're currently looking at
3038 * then we have a cross reference.
3039 */
3040 if (data_ref->root != root->root_key.objectid ||
3041 data_ref->objectid != objectid ||
3042 data_ref->offset != offset) {
3043 ret = 1;
3044 break;
3045 }
3046 }
3047 spin_unlock(&head->lock);
3048 mutex_unlock(&head->mutex);
ethanwu998ac6d2018-04-29 15:59:42 +08003049 btrfs_put_transaction(cur_trans);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003050 return ret;
3051}
3052
Liu Boe4c3b2d2017-01-30 12:25:28 -08003053static noinline int check_committed_ref(struct btrfs_root *root,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003054 struct btrfs_path *path,
3055 u64 objectid, u64 offset, u64 bytenr)
Chris Masonbe20aa92007-12-17 20:14:01 -05003056{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003057 struct btrfs_fs_info *fs_info = root->fs_info;
3058 struct btrfs_root *extent_root = fs_info->extent_root;
Yan Zhengf321e492008-07-30 09:26:11 -04003059 struct extent_buffer *leaf;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003060 struct btrfs_extent_data_ref *ref;
3061 struct btrfs_extent_inline_ref *iref;
3062 struct btrfs_extent_item *ei;
Chris Masonbe20aa92007-12-17 20:14:01 -05003063 struct btrfs_key key;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003064 u32 item_size;
Liu Bo3de28d52017-08-18 15:15:19 -06003065 int type;
Yan Zhengf321e492008-07-30 09:26:11 -04003066 int ret;
Chris Masonbe20aa92007-12-17 20:14:01 -05003067
Chris Masonbe20aa92007-12-17 20:14:01 -05003068 key.objectid = bytenr;
Zheng Yan31840ae2008-09-23 13:14:14 -04003069 key.offset = (u64)-1;
Yan Zhengf321e492008-07-30 09:26:11 -04003070 key.type = BTRFS_EXTENT_ITEM_KEY;
Chris Masonbe20aa92007-12-17 20:14:01 -05003071
Chris Masonbe20aa92007-12-17 20:14:01 -05003072 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
3073 if (ret < 0)
3074 goto out;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003075 BUG_ON(ret == 0); /* Corruption */
Yan Zheng80ff3852008-10-30 14:20:02 -04003076
3077 ret = -ENOENT;
3078 if (path->slots[0] == 0)
Zheng Yan31840ae2008-09-23 13:14:14 -04003079 goto out;
Chris Masonbe20aa92007-12-17 20:14:01 -05003080
Zheng Yan31840ae2008-09-23 13:14:14 -04003081 path->slots[0]--;
Yan Zhengf321e492008-07-30 09:26:11 -04003082 leaf = path->nodes[0];
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003083 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
Chris Masonbe20aa92007-12-17 20:14:01 -05003084
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003085 if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
Chris Masonbe20aa92007-12-17 20:14:01 -05003086 goto out;
Chris Masonbe20aa92007-12-17 20:14:01 -05003087
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003088 ret = 1;
3089 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003090 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
3091
3092 if (item_size != sizeof(*ei) +
3093 btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
3094 goto out;
3095
3096 if (btrfs_extent_generation(leaf, ei) <=
3097 btrfs_root_last_snapshot(&root->root_item))
3098 goto out;
3099
3100 iref = (struct btrfs_extent_inline_ref *)(ei + 1);
Liu Bo3de28d52017-08-18 15:15:19 -06003101
3102 type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA);
3103 if (type != BTRFS_EXTENT_DATA_REF_KEY)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003104 goto out;
3105
3106 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
3107 if (btrfs_extent_refs(leaf, ei) !=
3108 btrfs_extent_data_ref_count(leaf, ref) ||
3109 btrfs_extent_data_ref_root(leaf, ref) !=
3110 root->root_key.objectid ||
3111 btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
3112 btrfs_extent_data_ref_offset(leaf, ref) != offset)
3113 goto out;
3114
Yan Zhengf321e492008-07-30 09:26:11 -04003115 ret = 0;
Chris Masonbe20aa92007-12-17 20:14:01 -05003116out:
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003117 return ret;
3118}
3119
Liu Boe4c3b2d2017-01-30 12:25:28 -08003120int btrfs_cross_ref_exist(struct btrfs_root *root, u64 objectid, u64 offset,
3121 u64 bytenr)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003122{
3123 struct btrfs_path *path;
3124 int ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003125
3126 path = btrfs_alloc_path();
3127 if (!path)
Su Yue9132c4f2018-05-30 14:49:10 +08003128 return -ENOMEM;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003129
3130 do {
Liu Boe4c3b2d2017-01-30 12:25:28 -08003131 ret = check_committed_ref(root, path, objectid,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003132 offset, bytenr);
3133 if (ret && ret != -ENOENT)
3134 goto out;
3135
Misono Tomohiro380fd062018-08-30 10:59:16 +09003136 ret = check_delayed_ref(root, path, objectid, offset, bytenr);
3137 } while (ret == -EAGAIN);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003138
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003139out:
Yan Zhengf321e492008-07-30 09:26:11 -04003140 btrfs_free_path(path);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003141 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
3142 WARN_ON(ret > 0);
Yan Zhengf321e492008-07-30 09:26:11 -04003143 return ret;
3144}
3145
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003146static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
Chris Masonb7a9f292009-02-04 09:23:45 -05003147 struct btrfs_root *root,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003148 struct extent_buffer *buf,
Josef Bacike339a6b2014-07-02 10:54:25 -07003149 int full_backref, int inc)
Zheng Yan31840ae2008-09-23 13:14:14 -04003150{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003151 struct btrfs_fs_info *fs_info = root->fs_info;
Zheng Yan31840ae2008-09-23 13:14:14 -04003152 u64 bytenr;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003153 u64 num_bytes;
3154 u64 parent;
Zheng Yan31840ae2008-09-23 13:14:14 -04003155 u64 ref_root;
Zheng Yan31840ae2008-09-23 13:14:14 -04003156 u32 nritems;
Zheng Yan31840ae2008-09-23 13:14:14 -04003157 struct btrfs_key key;
3158 struct btrfs_file_extent_item *fi;
3159 int i;
3160 int level;
3161 int ret = 0;
David Sterbafccb84c2014-09-29 23:53:21 +02003162
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003163 if (btrfs_is_testing(fs_info))
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04003164 return 0;
David Sterbafccb84c2014-09-29 23:53:21 +02003165
Zheng Yan31840ae2008-09-23 13:14:14 -04003166 ref_root = btrfs_header_owner(buf);
Zheng Yan31840ae2008-09-23 13:14:14 -04003167 nritems = btrfs_header_nritems(buf);
3168 level = btrfs_header_level(buf);
3169
Miao Xie27cdeb72014-04-02 19:51:05 +08003170 if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003171 return 0;
Chris Masonb7a9f292009-02-04 09:23:45 -05003172
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003173 if (full_backref)
3174 parent = buf->start;
3175 else
3176 parent = 0;
3177
Zheng Yan31840ae2008-09-23 13:14:14 -04003178 for (i = 0; i < nritems; i++) {
Chris Masondb945352007-10-15 16:15:53 -04003179 if (level == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04003180 btrfs_item_key_to_cpu(buf, &key, i);
David Sterba962a2982014-06-04 18:41:45 +02003181 if (key.type != BTRFS_EXTENT_DATA_KEY)
Chris Mason54aa1f42007-06-22 14:16:25 -04003182 continue;
Chris Mason5f39d392007-10-15 16:14:19 -04003183 fi = btrfs_item_ptr(buf, i,
Chris Mason54aa1f42007-06-22 14:16:25 -04003184 struct btrfs_file_extent_item);
Chris Mason5f39d392007-10-15 16:14:19 -04003185 if (btrfs_file_extent_type(buf, fi) ==
Chris Mason54aa1f42007-06-22 14:16:25 -04003186 BTRFS_FILE_EXTENT_INLINE)
3187 continue;
Zheng Yan31840ae2008-09-23 13:14:14 -04003188 bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
3189 if (bytenr == 0)
Chris Mason54aa1f42007-06-22 14:16:25 -04003190 continue;
Zheng Yan31840ae2008-09-23 13:14:14 -04003191
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003192 num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
3193 key.offset -= btrfs_file_extent_offset(buf, fi);
Qu Wenruodd28b6a2019-04-04 14:45:30 +08003194 if (inc)
3195 ret = btrfs_inc_extent_ref(trans, root, bytenr,
3196 num_bytes, parent, ref_root,
3197 key.objectid, key.offset);
3198 else
3199 ret = btrfs_free_extent(trans, root, bytenr,
3200 num_bytes, parent, ref_root,
3201 key.objectid, key.offset);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003202 if (ret)
3203 goto fail;
Chris Masonb7a9f292009-02-04 09:23:45 -05003204 } else {
3205 bytenr = btrfs_node_blockptr(buf, i);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003206 num_bytes = fs_info->nodesize;
Qu Wenruodd28b6a2019-04-04 14:45:30 +08003207 if (inc)
3208 ret = btrfs_inc_extent_ref(trans, root, bytenr,
3209 num_bytes, parent, ref_root,
3210 level - 1, 0);
3211 else
3212 ret = btrfs_free_extent(trans, root, bytenr,
3213 num_bytes, parent, ref_root,
3214 level - 1, 0);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003215 if (ret)
Zheng Yan31840ae2008-09-23 13:14:14 -04003216 goto fail;
Chris Mason54aa1f42007-06-22 14:16:25 -04003217 }
3218 }
Zheng Yan31840ae2008-09-23 13:14:14 -04003219 return 0;
3220fail:
Chris Mason54aa1f42007-06-22 14:16:25 -04003221 return ret;
Chris Mason02217ed2007-03-02 16:08:05 -05003222}
3223
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003224int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
Josef Bacike339a6b2014-07-02 10:54:25 -07003225 struct extent_buffer *buf, int full_backref)
Zheng Yan31840ae2008-09-23 13:14:14 -04003226{
Josef Bacike339a6b2014-07-02 10:54:25 -07003227 return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003228}
Zheng Yan31840ae2008-09-23 13:14:14 -04003229
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003230int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
Josef Bacike339a6b2014-07-02 10:54:25 -07003231 struct extent_buffer *buf, int full_backref)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003232{
Josef Bacike339a6b2014-07-02 10:54:25 -07003233 return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003234}
3235
Chris Mason9078a3e2007-04-26 16:46:15 -04003236static int write_one_cache_group(struct btrfs_trans_handle *trans,
Chris Mason9078a3e2007-04-26 16:46:15 -04003237 struct btrfs_path *path,
3238 struct btrfs_block_group_cache *cache)
3239{
David Sterba39db2322019-03-20 11:57:46 +01003240 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Mason9078a3e2007-04-26 16:46:15 -04003241 int ret;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003242 struct btrfs_root *extent_root = fs_info->extent_root;
Chris Mason5f39d392007-10-15 16:14:19 -04003243 unsigned long bi;
3244 struct extent_buffer *leaf;
Chris Mason9078a3e2007-04-26 16:46:15 -04003245
Chris Mason9078a3e2007-04-26 16:46:15 -04003246 ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
Josef Bacikdf95e7f2014-12-12 16:02:20 -05003247 if (ret) {
3248 if (ret > 0)
3249 ret = -ENOENT;
Chris Mason54aa1f42007-06-22 14:16:25 -04003250 goto fail;
Josef Bacikdf95e7f2014-12-12 16:02:20 -05003251 }
Chris Mason5f39d392007-10-15 16:14:19 -04003252
3253 leaf = path->nodes[0];
3254 bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
3255 write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
3256 btrfs_mark_buffer_dirty(leaf);
Chris Mason54aa1f42007-06-22 14:16:25 -04003257fail:
Filipe Manana24b89d02015-04-25 18:31:05 +01003258 btrfs_release_path(path);
Josef Bacikdf95e7f2014-12-12 16:02:20 -05003259 return ret;
Chris Mason9078a3e2007-04-26 16:46:15 -04003260
3261}
3262
David Sterbaf87b7eb2019-03-20 12:01:07 +01003263static struct btrfs_block_group_cache *next_block_group(
3264 struct btrfs_block_group_cache *cache)
Yan Zheng4a8c9a62009-07-22 10:07:05 -04003265{
David Sterbaf87b7eb2019-03-20 12:01:07 +01003266 struct btrfs_fs_info *fs_info = cache->fs_info;
Yan Zheng4a8c9a62009-07-22 10:07:05 -04003267 struct rb_node *node;
Filipe Manana292cbd52014-11-26 15:28:50 +00003268
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003269 spin_lock(&fs_info->block_group_cache_lock);
Filipe Manana292cbd52014-11-26 15:28:50 +00003270
3271 /* If our block group was removed, we need a full search. */
3272 if (RB_EMPTY_NODE(&cache->cache_node)) {
3273 const u64 next_bytenr = cache->key.objectid + cache->key.offset;
3274
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003275 spin_unlock(&fs_info->block_group_cache_lock);
Filipe Manana292cbd52014-11-26 15:28:50 +00003276 btrfs_put_block_group(cache);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003277 cache = btrfs_lookup_first_block_group(fs_info, next_bytenr); return cache;
Filipe Manana292cbd52014-11-26 15:28:50 +00003278 }
Yan Zheng4a8c9a62009-07-22 10:07:05 -04003279 node = rb_next(&cache->cache_node);
3280 btrfs_put_block_group(cache);
3281 if (node) {
3282 cache = rb_entry(node, struct btrfs_block_group_cache,
3283 cache_node);
Josef Bacik11dfe352009-11-13 20:12:59 +00003284 btrfs_get_block_group(cache);
Yan Zheng4a8c9a62009-07-22 10:07:05 -04003285 } else
3286 cache = NULL;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003287 spin_unlock(&fs_info->block_group_cache_lock);
Yan Zheng4a8c9a62009-07-22 10:07:05 -04003288 return cache;
3289}
3290
Josef Bacik0af3d002010-06-21 14:48:16 -04003291static int cache_save_setup(struct btrfs_block_group_cache *block_group,
3292 struct btrfs_trans_handle *trans,
3293 struct btrfs_path *path)
3294{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003295 struct btrfs_fs_info *fs_info = block_group->fs_info;
3296 struct btrfs_root *root = fs_info->tree_root;
Josef Bacik0af3d002010-06-21 14:48:16 -04003297 struct inode *inode = NULL;
Qu Wenruo364ecf32017-02-27 15:10:38 +08003298 struct extent_changeset *data_reserved = NULL;
Josef Bacik0af3d002010-06-21 14:48:16 -04003299 u64 alloc_hint = 0;
Josef Bacik2b209822010-12-03 13:17:53 -05003300 int dcs = BTRFS_DC_ERROR;
David Sterbaf8c269d2015-01-16 17:21:12 +01003301 u64 num_pages = 0;
Josef Bacik0af3d002010-06-21 14:48:16 -04003302 int retries = 0;
3303 int ret = 0;
3304
3305 /*
3306 * If this block group is smaller than 100 megs don't bother caching the
3307 * block group.
3308 */
Byongho Leeee221842015-12-15 01:42:10 +09003309 if (block_group->key.offset < (100 * SZ_1M)) {
Josef Bacik0af3d002010-06-21 14:48:16 -04003310 spin_lock(&block_group->lock);
3311 block_group->disk_cache_state = BTRFS_DC_WRITTEN;
3312 spin_unlock(&block_group->lock);
3313 return 0;
3314 }
3315
Josef Bacik0c0ef4b2015-02-12 09:43:51 -05003316 if (trans->aborted)
3317 return 0;
Josef Bacik0af3d002010-06-21 14:48:16 -04003318again:
David Sterba7949f332019-03-20 13:40:19 +01003319 inode = lookup_free_space_inode(block_group, path);
Josef Bacik0af3d002010-06-21 14:48:16 -04003320 if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
3321 ret = PTR_ERR(inode);
David Sterbab3b4aa72011-04-21 01:20:15 +02003322 btrfs_release_path(path);
Josef Bacik0af3d002010-06-21 14:48:16 -04003323 goto out;
3324 }
3325
3326 if (IS_ERR(inode)) {
3327 BUG_ON(retries);
3328 retries++;
3329
3330 if (block_group->ro)
3331 goto out_free;
3332
David Sterba4ca75f12019-03-20 13:42:57 +01003333 ret = create_free_space_inode(trans, block_group, path);
Josef Bacik0af3d002010-06-21 14:48:16 -04003334 if (ret)
3335 goto out_free;
3336 goto again;
3337 }
3338
3339 /*
3340 * We want to set the generation to 0, that way if anything goes wrong
3341 * from here on out we know not to trust this cache when we load up next
3342 * time.
3343 */
3344 BTRFS_I(inode)->generation = 0;
3345 ret = btrfs_update_inode(trans, root, inode);
Josef Bacik0c0ef4b2015-02-12 09:43:51 -05003346 if (ret) {
3347 /*
3348 * So theoretically we could recover from this, simply set the
3349 * super cache generation to 0 so we know to invalidate the
3350 * cache, but then we'd have to keep track of the block groups
3351 * that fail this way so we know we _have_ to reset this cache
3352 * before the next commit or risk reading stale cache. So to
3353 * limit our exposure to horrible edge cases lets just abort the
3354 * transaction, this only happens in really bad situations
3355 * anyway.
3356 */
Jeff Mahoney66642832016-06-10 18:19:25 -04003357 btrfs_abort_transaction(trans, ret);
Josef Bacik0c0ef4b2015-02-12 09:43:51 -05003358 goto out_put;
3359 }
Josef Bacik0af3d002010-06-21 14:48:16 -04003360 WARN_ON(ret);
3361
Josef Bacik8e138e02017-11-17 14:50:46 -05003362 /* We've already setup this transaction, go ahead and exit */
3363 if (block_group->cache_generation == trans->transid &&
3364 i_size_read(inode)) {
3365 dcs = BTRFS_DC_SETUP;
3366 goto out_put;
3367 }
3368
Josef Bacik0af3d002010-06-21 14:48:16 -04003369 if (i_size_read(inode) > 0) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04003370 ret = btrfs_check_trunc_cache_free_space(fs_info,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003371 &fs_info->global_block_rsv);
Miao Xie7b61cd92013-05-13 13:55:09 +00003372 if (ret)
3373 goto out_put;
3374
Jeff Mahoney77ab86b2017-02-15 16:28:30 -05003375 ret = btrfs_truncate_free_space_cache(trans, NULL, inode);
Josef Bacik0af3d002010-06-21 14:48:16 -04003376 if (ret)
3377 goto out_put;
3378 }
3379
3380 spin_lock(&block_group->lock);
Liu Bocf7c1ef2012-07-06 03:31:34 -06003381 if (block_group->cached != BTRFS_CACHE_FINISHED ||
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003382 !btrfs_test_opt(fs_info, SPACE_CACHE)) {
Liu Bocf7c1ef2012-07-06 03:31:34 -06003383 /*
3384 * don't bother trying to write stuff out _if_
3385 * a) we're not cached,
Liu Bo1a79c1f2017-03-06 13:49:02 -08003386 * b) we're with nospace_cache mount option,
3387 * c) we're with v2 space_cache (FREE_SPACE_TREE).
Liu Bocf7c1ef2012-07-06 03:31:34 -06003388 */
Josef Bacik2b209822010-12-03 13:17:53 -05003389 dcs = BTRFS_DC_WRITTEN;
Josef Bacik0af3d002010-06-21 14:48:16 -04003390 spin_unlock(&block_group->lock);
3391 goto out_put;
3392 }
3393 spin_unlock(&block_group->lock);
3394
Josef Bacik6fc823b2012-08-06 13:46:38 -06003395 /*
Josef Bacik2968b1f2015-10-01 12:55:18 -04003396 * We hit an ENOSPC when setting up the cache in this transaction, just
3397 * skip doing the setup, we've already cleared the cache so we're safe.
3398 */
3399 if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) {
3400 ret = -ENOSPC;
3401 goto out_put;
3402 }
3403
3404 /*
Josef Bacik6fc823b2012-08-06 13:46:38 -06003405 * Try to preallocate enough space based on how big the block group is.
3406 * Keep in mind this has to include any pinned space which could end up
3407 * taking up quite a bit since it's not folded into the other space
3408 * cache.
3409 */
Byongho Leeee221842015-12-15 01:42:10 +09003410 num_pages = div_u64(block_group->key.offset, SZ_256M);
Josef Bacik0af3d002010-06-21 14:48:16 -04003411 if (!num_pages)
3412 num_pages = 1;
3413
Josef Bacik0af3d002010-06-21 14:48:16 -04003414 num_pages *= 16;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003415 num_pages *= PAGE_SIZE;
Josef Bacik0af3d002010-06-21 14:48:16 -04003416
Qu Wenruo364ecf32017-02-27 15:10:38 +08003417 ret = btrfs_check_data_free_space(inode, &data_reserved, 0, num_pages);
Josef Bacik0af3d002010-06-21 14:48:16 -04003418 if (ret)
3419 goto out_put;
3420
3421 ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
3422 num_pages, num_pages,
3423 &alloc_hint);
Josef Bacik2968b1f2015-10-01 12:55:18 -04003424 /*
3425 * Our cache requires contiguous chunks so that we don't modify a bunch
3426 * of metadata or split extents when writing the cache out, which means
3427 * we can enospc if we are heavily fragmented in addition to just normal
3428 * out of space conditions. So if we hit this just skip setting up any
3429 * other block groups for this transaction, maybe we'll unpin enough
3430 * space the next time around.
3431 */
Josef Bacik2b209822010-12-03 13:17:53 -05003432 if (!ret)
3433 dcs = BTRFS_DC_SETUP;
Josef Bacik2968b1f2015-10-01 12:55:18 -04003434 else if (ret == -ENOSPC)
3435 set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags);
Josef Bacikc09544e2011-08-30 10:19:10 -04003436
Josef Bacik0af3d002010-06-21 14:48:16 -04003437out_put:
3438 iput(inode);
3439out_free:
David Sterbab3b4aa72011-04-21 01:20:15 +02003440 btrfs_release_path(path);
Josef Bacik0af3d002010-06-21 14:48:16 -04003441out:
3442 spin_lock(&block_group->lock);
Josef Bacike65cbb92011-12-13 16:04:54 -05003443 if (!ret && dcs == BTRFS_DC_SETUP)
Josef Bacik5b0e95b2011-10-06 08:58:24 -04003444 block_group->cache_generation = trans->transid;
Josef Bacik2b209822010-12-03 13:17:53 -05003445 block_group->disk_cache_state = dcs;
Josef Bacik0af3d002010-06-21 14:48:16 -04003446 spin_unlock(&block_group->lock);
3447
Qu Wenruo364ecf32017-02-27 15:10:38 +08003448 extent_changeset_free(data_reserved);
Josef Bacik0af3d002010-06-21 14:48:16 -04003449 return ret;
3450}
3451
David Sterbabbebb3e2019-03-20 12:02:55 +01003452int btrfs_setup_space_cache(struct btrfs_trans_handle *trans)
Josef Bacikdcdf7f62015-03-02 16:37:31 -05003453{
David Sterbabbebb3e2019-03-20 12:02:55 +01003454 struct btrfs_fs_info *fs_info = trans->fs_info;
Josef Bacikdcdf7f62015-03-02 16:37:31 -05003455 struct btrfs_block_group_cache *cache, *tmp;
3456 struct btrfs_transaction *cur_trans = trans->transaction;
3457 struct btrfs_path *path;
3458
3459 if (list_empty(&cur_trans->dirty_bgs) ||
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003460 !btrfs_test_opt(fs_info, SPACE_CACHE))
Josef Bacikdcdf7f62015-03-02 16:37:31 -05003461 return 0;
3462
3463 path = btrfs_alloc_path();
3464 if (!path)
3465 return -ENOMEM;
3466
3467 /* Could add new block groups, use _safe just in case */
3468 list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs,
3469 dirty_list) {
3470 if (cache->disk_cache_state == BTRFS_DC_CLEAR)
3471 cache_save_setup(cache, trans, path);
3472 }
3473
3474 btrfs_free_path(path);
3475 return 0;
3476}
3477
Chris Mason1bbc6212015-04-06 12:46:08 -07003478/*
3479 * transaction commit does final block group cache writeback during a
3480 * critical section where nothing is allowed to change the FS. This is
3481 * required in order for the cache to actually match the block group,
3482 * but can introduce a lot of latency into the commit.
3483 *
3484 * So, btrfs_start_dirty_block_groups is here to kick off block group
3485 * cache IO. There's a chance we'll have to redo some of it if the
3486 * block group changes again during the commit, but it greatly reduces
3487 * the commit latency by getting rid of the easy block groups while
3488 * we're still allowing others to join the commit.
3489 */
Nikolay Borisov21217052018-02-07 17:55:41 +02003490int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
Chris Mason1bbc6212015-04-06 12:46:08 -07003491{
Nikolay Borisov21217052018-02-07 17:55:41 +02003492 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Mason1bbc6212015-04-06 12:46:08 -07003493 struct btrfs_block_group_cache *cache;
3494 struct btrfs_transaction *cur_trans = trans->transaction;
3495 int ret = 0;
3496 int should_put;
3497 struct btrfs_path *path = NULL;
3498 LIST_HEAD(dirty);
3499 struct list_head *io = &cur_trans->io_bgs;
3500 int num_started = 0;
3501 int loops = 0;
3502
3503 spin_lock(&cur_trans->dirty_bgs_lock);
Filipe Mananab58d1a92015-04-25 18:29:16 +01003504 if (list_empty(&cur_trans->dirty_bgs)) {
3505 spin_unlock(&cur_trans->dirty_bgs_lock);
3506 return 0;
Chris Mason1bbc6212015-04-06 12:46:08 -07003507 }
Filipe Mananab58d1a92015-04-25 18:29:16 +01003508 list_splice_init(&cur_trans->dirty_bgs, &dirty);
Chris Mason1bbc6212015-04-06 12:46:08 -07003509 spin_unlock(&cur_trans->dirty_bgs_lock);
3510
3511again:
Chris Mason1bbc6212015-04-06 12:46:08 -07003512 /*
3513 * make sure all the block groups on our dirty list actually
3514 * exist
3515 */
Nikolay Borisov6c686b32018-02-07 17:55:40 +02003516 btrfs_create_pending_block_groups(trans);
Chris Mason1bbc6212015-04-06 12:46:08 -07003517
3518 if (!path) {
3519 path = btrfs_alloc_path();
3520 if (!path)
3521 return -ENOMEM;
3522 }
3523
Filipe Mananab58d1a92015-04-25 18:29:16 +01003524 /*
3525 * cache_write_mutex is here only to save us from balance or automatic
3526 * removal of empty block groups deleting this block group while we are
3527 * writing out the cache
3528 */
3529 mutex_lock(&trans->transaction->cache_write_mutex);
Chris Mason1bbc6212015-04-06 12:46:08 -07003530 while (!list_empty(&dirty)) {
Josef Bacikba2c4d42018-12-03 10:20:33 -05003531 bool drop_reserve = true;
3532
Chris Mason1bbc6212015-04-06 12:46:08 -07003533 cache = list_first_entry(&dirty,
3534 struct btrfs_block_group_cache,
3535 dirty_list);
Chris Mason1bbc6212015-04-06 12:46:08 -07003536 /*
3537 * this can happen if something re-dirties a block
3538 * group that is already under IO. Just wait for it to
3539 * finish and then do it all again
3540 */
3541 if (!list_empty(&cache->io_list)) {
3542 list_del_init(&cache->io_list);
Jeff Mahoneyafdb5712016-09-09 12:09:35 -04003543 btrfs_wait_cache_io(trans, cache, path);
Chris Mason1bbc6212015-04-06 12:46:08 -07003544 btrfs_put_block_group(cache);
3545 }
3546
3547
3548 /*
3549 * btrfs_wait_cache_io uses the cache->dirty_list to decide
3550 * if it should update the cache_state. Don't delete
3551 * until after we wait.
3552 *
3553 * Since we're not running in the commit critical section
3554 * we need the dirty_bgs_lock to protect from update_block_group
3555 */
3556 spin_lock(&cur_trans->dirty_bgs_lock);
3557 list_del_init(&cache->dirty_list);
3558 spin_unlock(&cur_trans->dirty_bgs_lock);
3559
3560 should_put = 1;
3561
3562 cache_save_setup(cache, trans, path);
3563
3564 if (cache->disk_cache_state == BTRFS_DC_SETUP) {
3565 cache->io_ctl.inode = NULL;
David Sterbafe041532019-03-20 13:51:56 +01003566 ret = btrfs_write_out_cache(trans, cache, path);
Chris Mason1bbc6212015-04-06 12:46:08 -07003567 if (ret == 0 && cache->io_ctl.inode) {
3568 num_started++;
3569 should_put = 0;
3570
3571 /*
Nikolay Borisov45ae2c12018-02-08 18:25:18 +02003572 * The cache_write_mutex is protecting the
3573 * io_list, also refer to the definition of
3574 * btrfs_transaction::io_bgs for more details
Chris Mason1bbc6212015-04-06 12:46:08 -07003575 */
3576 list_add_tail(&cache->io_list, io);
3577 } else {
3578 /*
3579 * if we failed to write the cache, the
3580 * generation will be bad and life goes on
3581 */
3582 ret = 0;
3583 }
3584 }
Filipe Mananaff1f8252015-05-06 16:15:09 +01003585 if (!ret) {
David Sterba39db2322019-03-20 11:57:46 +01003586 ret = write_one_cache_group(trans, path, cache);
Filipe Mananaff1f8252015-05-06 16:15:09 +01003587 /*
3588 * Our block group might still be attached to the list
3589 * of new block groups in the transaction handle of some
3590 * other task (struct btrfs_trans_handle->new_bgs). This
3591 * means its block group item isn't yet in the extent
3592 * tree. If this happens ignore the error, as we will
3593 * try again later in the critical section of the
3594 * transaction commit.
3595 */
3596 if (ret == -ENOENT) {
3597 ret = 0;
3598 spin_lock(&cur_trans->dirty_bgs_lock);
3599 if (list_empty(&cache->dirty_list)) {
3600 list_add_tail(&cache->dirty_list,
3601 &cur_trans->dirty_bgs);
3602 btrfs_get_block_group(cache);
Josef Bacikba2c4d42018-12-03 10:20:33 -05003603 drop_reserve = false;
Filipe Mananaff1f8252015-05-06 16:15:09 +01003604 }
3605 spin_unlock(&cur_trans->dirty_bgs_lock);
3606 } else if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04003607 btrfs_abort_transaction(trans, ret);
Filipe Mananaff1f8252015-05-06 16:15:09 +01003608 }
3609 }
Chris Mason1bbc6212015-04-06 12:46:08 -07003610
Andrea Gelmini52042d82018-11-28 12:05:13 +01003611 /* if it's not on the io list, we need to put the block group */
Chris Mason1bbc6212015-04-06 12:46:08 -07003612 if (should_put)
3613 btrfs_put_block_group(cache);
Josef Bacikba2c4d42018-12-03 10:20:33 -05003614 if (drop_reserve)
3615 btrfs_delayed_refs_rsv_release(fs_info, 1);
Chris Mason1bbc6212015-04-06 12:46:08 -07003616
3617 if (ret)
3618 break;
Filipe Mananab58d1a92015-04-25 18:29:16 +01003619
3620 /*
3621 * Avoid blocking other tasks for too long. It might even save
3622 * us from writing caches for block groups that are going to be
3623 * removed.
3624 */
3625 mutex_unlock(&trans->transaction->cache_write_mutex);
3626 mutex_lock(&trans->transaction->cache_write_mutex);
Chris Mason1bbc6212015-04-06 12:46:08 -07003627 }
Filipe Mananab58d1a92015-04-25 18:29:16 +01003628 mutex_unlock(&trans->transaction->cache_write_mutex);
Chris Mason1bbc6212015-04-06 12:46:08 -07003629
3630 /*
3631 * go through delayed refs for all the stuff we've just kicked off
3632 * and then loop back (just once)
3633 */
Nikolay Borisovc79a70b2018-03-15 17:27:37 +02003634 ret = btrfs_run_delayed_refs(trans, 0);
Chris Mason1bbc6212015-04-06 12:46:08 -07003635 if (!ret && loops == 0) {
3636 loops++;
3637 spin_lock(&cur_trans->dirty_bgs_lock);
3638 list_splice_init(&cur_trans->dirty_bgs, &dirty);
Filipe Mananab58d1a92015-04-25 18:29:16 +01003639 /*
3640 * dirty_bgs_lock protects us from concurrent block group
3641 * deletes too (not just cache_write_mutex).
3642 */
3643 if (!list_empty(&dirty)) {
3644 spin_unlock(&cur_trans->dirty_bgs_lock);
3645 goto again;
3646 }
Chris Mason1bbc6212015-04-06 12:46:08 -07003647 spin_unlock(&cur_trans->dirty_bgs_lock);
Liu Boc79a1752016-07-20 17:44:12 -07003648 } else if (ret < 0) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04003649 btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
Chris Mason1bbc6212015-04-06 12:46:08 -07003650 }
3651
3652 btrfs_free_path(path);
3653 return ret;
3654}
3655
David Sterba5742d152019-03-20 12:04:08 +01003656int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans)
Chris Mason9078a3e2007-04-26 16:46:15 -04003657{
David Sterba5742d152019-03-20 12:04:08 +01003658 struct btrfs_fs_info *fs_info = trans->fs_info;
Yan Zheng4a8c9a62009-07-22 10:07:05 -04003659 struct btrfs_block_group_cache *cache;
Josef Bacikce93ec52014-11-17 15:45:48 -05003660 struct btrfs_transaction *cur_trans = trans->transaction;
3661 int ret = 0;
Chris Masonc9dc4c62015-04-04 17:14:42 -07003662 int should_put;
Chris Mason9078a3e2007-04-26 16:46:15 -04003663 struct btrfs_path *path;
Chris Mason1bbc6212015-04-06 12:46:08 -07003664 struct list_head *io = &cur_trans->io_bgs;
Chris Masonc9dc4c62015-04-04 17:14:42 -07003665 int num_started = 0;
Chris Mason9078a3e2007-04-26 16:46:15 -04003666
3667 path = btrfs_alloc_path();
3668 if (!path)
3669 return -ENOMEM;
3670
Josef Bacikce93ec52014-11-17 15:45:48 -05003671 /*
Filipe Mananae44081e2015-12-18 03:02:48 +00003672 * Even though we are in the critical section of the transaction commit,
3673 * we can still have concurrent tasks adding elements to this
3674 * transaction's list of dirty block groups. These tasks correspond to
3675 * endio free space workers started when writeback finishes for a
3676 * space cache, which run inode.c:btrfs_finish_ordered_io(), and can
3677 * allocate new block groups as a result of COWing nodes of the root
3678 * tree when updating the free space inode. The writeback for the space
3679 * caches is triggered by an earlier call to
3680 * btrfs_start_dirty_block_groups() and iterations of the following
3681 * loop.
3682 * Also we want to do the cache_save_setup first and then run the
Josef Bacikce93ec52014-11-17 15:45:48 -05003683 * delayed refs to make sure we have the best chance at doing this all
3684 * in one shot.
3685 */
Filipe Mananae44081e2015-12-18 03:02:48 +00003686 spin_lock(&cur_trans->dirty_bgs_lock);
Josef Bacikce93ec52014-11-17 15:45:48 -05003687 while (!list_empty(&cur_trans->dirty_bgs)) {
3688 cache = list_first_entry(&cur_trans->dirty_bgs,
3689 struct btrfs_block_group_cache,
3690 dirty_list);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003691
3692 /*
3693 * this can happen if cache_save_setup re-dirties a block
3694 * group that is already under IO. Just wait for it to
3695 * finish and then do it all again
3696 */
3697 if (!list_empty(&cache->io_list)) {
Filipe Mananae44081e2015-12-18 03:02:48 +00003698 spin_unlock(&cur_trans->dirty_bgs_lock);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003699 list_del_init(&cache->io_list);
Jeff Mahoneyafdb5712016-09-09 12:09:35 -04003700 btrfs_wait_cache_io(trans, cache, path);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003701 btrfs_put_block_group(cache);
Filipe Mananae44081e2015-12-18 03:02:48 +00003702 spin_lock(&cur_trans->dirty_bgs_lock);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003703 }
3704
Chris Mason1bbc6212015-04-06 12:46:08 -07003705 /*
3706 * don't remove from the dirty list until after we've waited
3707 * on any pending IO
3708 */
Josef Bacikce93ec52014-11-17 15:45:48 -05003709 list_del_init(&cache->dirty_list);
Filipe Mananae44081e2015-12-18 03:02:48 +00003710 spin_unlock(&cur_trans->dirty_bgs_lock);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003711 should_put = 1;
3712
Chris Mason1bbc6212015-04-06 12:46:08 -07003713 cache_save_setup(cache, trans, path);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003714
Josef Bacikce93ec52014-11-17 15:45:48 -05003715 if (!ret)
Nikolay Borisovc79a70b2018-03-15 17:27:37 +02003716 ret = btrfs_run_delayed_refs(trans,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04003717 (unsigned long) -1);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003718
3719 if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) {
3720 cache->io_ctl.inode = NULL;
David Sterbafe041532019-03-20 13:51:56 +01003721 ret = btrfs_write_out_cache(trans, cache, path);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003722 if (ret == 0 && cache->io_ctl.inode) {
3723 num_started++;
3724 should_put = 0;
Chris Mason1bbc6212015-04-06 12:46:08 -07003725 list_add_tail(&cache->io_list, io);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003726 } else {
3727 /*
3728 * if we failed to write the cache, the
3729 * generation will be bad and life goes on
3730 */
3731 ret = 0;
3732 }
3733 }
Filipe Mananaff1f8252015-05-06 16:15:09 +01003734 if (!ret) {
David Sterba39db2322019-03-20 11:57:46 +01003735 ret = write_one_cache_group(trans, path, cache);
Filipe Manana2bc0bb52015-12-30 02:42:30 +00003736 /*
3737 * One of the free space endio workers might have
3738 * created a new block group while updating a free space
3739 * cache's inode (at inode.c:btrfs_finish_ordered_io())
3740 * and hasn't released its transaction handle yet, in
3741 * which case the new block group is still attached to
3742 * its transaction handle and its creation has not
3743 * finished yet (no block group item in the extent tree
3744 * yet, etc). If this is the case, wait for all free
3745 * space endio workers to finish and retry. This is a
3746 * a very rare case so no need for a more efficient and
3747 * complex approach.
3748 */
3749 if (ret == -ENOENT) {
3750 wait_event(cur_trans->writer_wait,
3751 atomic_read(&cur_trans->num_writers) == 1);
David Sterba39db2322019-03-20 11:57:46 +01003752 ret = write_one_cache_group(trans, path, cache);
Filipe Manana2bc0bb52015-12-30 02:42:30 +00003753 }
Filipe Mananaff1f8252015-05-06 16:15:09 +01003754 if (ret)
Jeff Mahoney66642832016-06-10 18:19:25 -04003755 btrfs_abort_transaction(trans, ret);
Filipe Mananaff1f8252015-05-06 16:15:09 +01003756 }
Chris Masonc9dc4c62015-04-04 17:14:42 -07003757
3758 /* if its not on the io list, we need to put the block group */
3759 if (should_put)
3760 btrfs_put_block_group(cache);
Josef Bacikba2c4d42018-12-03 10:20:33 -05003761 btrfs_delayed_refs_rsv_release(fs_info, 1);
Filipe Mananae44081e2015-12-18 03:02:48 +00003762 spin_lock(&cur_trans->dirty_bgs_lock);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003763 }
Filipe Mananae44081e2015-12-18 03:02:48 +00003764 spin_unlock(&cur_trans->dirty_bgs_lock);
Chris Masonc9dc4c62015-04-04 17:14:42 -07003765
Nikolay Borisov45ae2c12018-02-08 18:25:18 +02003766 /*
3767 * Refer to the definition of io_bgs member for details why it's safe
3768 * to use it without any locking
3769 */
Chris Mason1bbc6212015-04-06 12:46:08 -07003770 while (!list_empty(io)) {
3771 cache = list_first_entry(io, struct btrfs_block_group_cache,
Chris Masonc9dc4c62015-04-04 17:14:42 -07003772 io_list);
3773 list_del_init(&cache->io_list);
Jeff Mahoneyafdb5712016-09-09 12:09:35 -04003774 btrfs_wait_cache_io(trans, cache, path);
Josef Bacik0af3d002010-06-21 14:48:16 -04003775 btrfs_put_block_group(cache);
3776 }
3777
Chris Mason9078a3e2007-04-26 16:46:15 -04003778 btrfs_free_path(path);
Josef Bacikce93ec52014-11-17 15:45:48 -05003779 return ret;
Chris Mason9078a3e2007-04-26 16:46:15 -04003780}
3781
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04003782int btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)
Yan Zhengd2fb3432008-12-11 16:30:39 -05003783{
3784 struct btrfs_block_group_cache *block_group;
3785 int readonly = 0;
3786
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003787 block_group = btrfs_lookup_block_group(fs_info, bytenr);
Yan Zhengd2fb3432008-12-11 16:30:39 -05003788 if (!block_group || block_group->ro)
3789 readonly = 1;
3790 if (block_group)
Chris Masonfa9c0d792009-04-03 09:47:43 -04003791 btrfs_put_block_group(block_group);
Yan Zhengd2fb3432008-12-11 16:30:39 -05003792 return readonly;
3793}
3794
Filipe Mananaf78c4362016-05-09 13:15:41 +01003795bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
3796{
3797 struct btrfs_block_group_cache *bg;
3798 bool ret = true;
3799
3800 bg = btrfs_lookup_block_group(fs_info, bytenr);
3801 if (!bg)
3802 return false;
3803
3804 spin_lock(&bg->lock);
3805 if (bg->ro)
3806 ret = false;
3807 else
3808 atomic_inc(&bg->nocow_writers);
3809 spin_unlock(&bg->lock);
3810
3811 /* no put on block group, done by btrfs_dec_nocow_writers */
3812 if (!ret)
3813 btrfs_put_block_group(bg);
3814
3815 return ret;
3816
3817}
3818
3819void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
3820{
3821 struct btrfs_block_group_cache *bg;
3822
3823 bg = btrfs_lookup_block_group(fs_info, bytenr);
3824 ASSERT(bg);
3825 if (atomic_dec_and_test(&bg->nocow_writers))
Peter Zijlstra46259562018-03-15 11:43:08 +01003826 wake_up_var(&bg->nocow_writers);
Filipe Mananaf78c4362016-05-09 13:15:41 +01003827 /*
3828 * Once for our lookup and once for the lookup done by a previous call
3829 * to btrfs_inc_nocow_writers()
3830 */
3831 btrfs_put_block_group(bg);
3832 btrfs_put_block_group(bg);
3833}
3834
Filipe Mananaf78c4362016-05-09 13:15:41 +01003835void btrfs_wait_nocow_writers(struct btrfs_block_group_cache *bg)
3836{
Peter Zijlstra46259562018-03-15 11:43:08 +01003837 wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers));
Filipe Mananaf78c4362016-05-09 13:15:41 +01003838}
3839
Jeff Mahoney6ab0a202013-11-01 13:07:04 -04003840static const char *alloc_name(u64 flags)
3841{
3842 switch (flags) {
3843 case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA:
3844 return "mixed";
3845 case BTRFS_BLOCK_GROUP_METADATA:
3846 return "metadata";
3847 case BTRFS_BLOCK_GROUP_DATA:
3848 return "data";
3849 case BTRFS_BLOCK_GROUP_SYSTEM:
3850 return "system";
3851 default:
3852 WARN_ON(1);
3853 return "invalid-combination";
3854 };
3855}
3856
Lu Fengqi4ca61682018-05-28 14:30:27 +08003857static int create_space_info(struct btrfs_fs_info *info, u64 flags)
Nikolay Borisov2be12ef2017-05-22 09:35:49 +03003858{
3859
3860 struct btrfs_space_info *space_info;
3861 int i;
3862 int ret;
3863
3864 space_info = kzalloc(sizeof(*space_info), GFP_NOFS);
3865 if (!space_info)
3866 return -ENOMEM;
3867
3868 ret = percpu_counter_init(&space_info->total_bytes_pinned, 0,
3869 GFP_KERNEL);
3870 if (ret) {
3871 kfree(space_info);
3872 return ret;
3873 }
3874
3875 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
3876 INIT_LIST_HEAD(&space_info->block_groups[i]);
3877 init_rwsem(&space_info->groups_sem);
3878 spin_lock_init(&space_info->lock);
3879 space_info->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3880 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3881 init_waitqueue_head(&space_info->wait);
3882 INIT_LIST_HEAD(&space_info->ro_bgs);
3883 INIT_LIST_HEAD(&space_info->tickets);
3884 INIT_LIST_HEAD(&space_info->priority_tickets);
3885
3886 ret = kobject_init_and_add(&space_info->kobj, &space_info_ktype,
3887 info->space_info_kobj, "%s",
3888 alloc_name(space_info->flags));
3889 if (ret) {
3890 percpu_counter_destroy(&space_info->total_bytes_pinned);
3891 kfree(space_info);
3892 return ret;
3893 }
3894
Nikolay Borisov2be12ef2017-05-22 09:35:49 +03003895 list_add_rcu(&space_info->list, &info->space_info);
3896 if (flags & BTRFS_BLOCK_GROUP_DATA)
3897 info->data_sinfo = space_info;
3898
3899 return ret;
3900}
3901
Nikolay Borisovd2006e62017-05-22 09:35:50 +03003902static void update_space_info(struct btrfs_fs_info *info, u64 flags,
Chris Mason593060d2008-03-25 16:50:33 -04003903 u64 total_bytes, u64 bytes_used,
Josef Bacike40edf22016-03-25 13:25:47 -04003904 u64 bytes_readonly,
Chris Mason593060d2008-03-25 16:50:33 -04003905 struct btrfs_space_info **space_info)
3906{
3907 struct btrfs_space_info *found;
Yan, Zhengb742bb822010-05-16 10:46:24 -04003908 int factor;
3909
David Sterba46df06b2018-07-13 20:46:30 +02003910 factor = btrfs_bg_type_to_factor(flags);
Chris Mason593060d2008-03-25 16:50:33 -04003911
3912 found = __find_space_info(info, flags);
Nikolay Borisovd2006e62017-05-22 09:35:50 +03003913 ASSERT(found);
3914 spin_lock(&found->lock);
3915 found->total_bytes += total_bytes;
3916 found->disk_total += total_bytes * factor;
3917 found->bytes_used += bytes_used;
3918 found->disk_used += bytes_used * factor;
3919 found->bytes_readonly += bytes_readonly;
3920 if (total_bytes > 0)
3921 found->full = 0;
3922 space_info_add_new_bytes(info, found, total_bytes -
3923 bytes_used - bytes_readonly);
3924 spin_unlock(&found->lock);
3925 *space_info = found;
Chris Mason593060d2008-03-25 16:50:33 -04003926}
3927
Chris Mason8790d502008-04-03 16:29:03 -04003928static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
3929{
Ilya Dryomov899c81e2012-03-27 17:09:16 +03003930 u64 extra_flags = chunk_to_extended(flags) &
3931 BTRFS_EXTENDED_PROFILE_MASK;
Ilya Dryomova46d11a2012-01-16 22:04:47 +02003932
Miao Xiede98ced2013-01-29 10:13:12 +00003933 write_seqlock(&fs_info->profiles_lock);
Ilya Dryomova46d11a2012-01-16 22:04:47 +02003934 if (flags & BTRFS_BLOCK_GROUP_DATA)
3935 fs_info->avail_data_alloc_bits |= extra_flags;
3936 if (flags & BTRFS_BLOCK_GROUP_METADATA)
3937 fs_info->avail_metadata_alloc_bits |= extra_flags;
3938 if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3939 fs_info->avail_system_alloc_bits |= extra_flags;
Miao Xiede98ced2013-01-29 10:13:12 +00003940 write_sequnlock(&fs_info->profiles_lock);
Chris Mason8790d502008-04-03 16:29:03 -04003941}
Chris Mason593060d2008-03-25 16:50:33 -04003942
Ilya Dryomova46d11a2012-01-16 22:04:47 +02003943/*
Ilya Dryomovfc67c452012-03-27 17:09:17 +03003944 * returns target flags in extended format or 0 if restripe for this
3945 * chunk_type is not in progress
Ilya Dryomovc6664b42012-04-12 16:03:56 -04003946 *
David Sterbadccdb072018-03-21 00:20:05 +01003947 * should be called with balance_lock held
Ilya Dryomovfc67c452012-03-27 17:09:17 +03003948 */
3949static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
3950{
3951 struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3952 u64 target = 0;
3953
Ilya Dryomovfc67c452012-03-27 17:09:17 +03003954 if (!bctl)
3955 return 0;
3956
3957 if (flags & BTRFS_BLOCK_GROUP_DATA &&
3958 bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3959 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
3960 } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
3961 bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3962 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
3963 } else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
3964 bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3965 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
3966 }
3967
3968 return target;
3969}
3970
3971/*
Ilya Dryomova46d11a2012-01-16 22:04:47 +02003972 * @flags: available profiles in extended format (see ctree.h)
3973 *
Ilya Dryomove4d8ec02012-01-16 22:04:48 +02003974 * Returns reduced profile in chunk format. If profile changing is in
3975 * progress (either running or paused) picks the target profile (if it's
3976 * already available), otherwise falls back to plain reducing.
Ilya Dryomova46d11a2012-01-16 22:04:47 +02003977 */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04003978static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags)
Chris Masonec44a352008-04-28 15:29:52 -04003979{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003980 u64 num_devices = fs_info->fs_devices->rw_devices;
Ilya Dryomovfc67c452012-03-27 17:09:17 +03003981 u64 target;
Zhao Lei9c170b22015-09-15 21:08:08 +08003982 u64 raid_type;
3983 u64 allowed = 0;
Chris Masona061fc82008-05-07 11:43:44 -04003984
Ilya Dryomovfc67c452012-03-27 17:09:17 +03003985 /*
3986 * see if restripe for this chunk_type is in progress, if so
3987 * try to reduce to the target profile
3988 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003989 spin_lock(&fs_info->balance_lock);
3990 target = get_restripe_target(fs_info, flags);
Ilya Dryomovfc67c452012-03-27 17:09:17 +03003991 if (target) {
3992 /* pick target profile only if it's already available */
3993 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003994 spin_unlock(&fs_info->balance_lock);
Ilya Dryomovfc67c452012-03-27 17:09:17 +03003995 return extended_to_chunk(target);
Ilya Dryomove4d8ec02012-01-16 22:04:48 +02003996 }
3997 }
Jeff Mahoney0b246af2016-06-22 18:54:23 -04003998 spin_unlock(&fs_info->balance_lock);
Ilya Dryomove4d8ec02012-01-16 22:04:48 +02003999
David Woodhouse53b381b2013-01-29 18:40:14 -05004000 /* First, mask out the RAID levels which aren't possible */
Zhao Lei9c170b22015-09-15 21:08:08 +08004001 for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
4002 if (num_devices >= btrfs_raid_array[raid_type].devs_min)
Anand Jain41a6e892018-04-25 19:01:43 +08004003 allowed |= btrfs_raid_array[raid_type].bg_flag;
Zhao Lei9c170b22015-09-15 21:08:08 +08004004 }
4005 allowed &= flags;
Chris Masona061fc82008-05-07 11:43:44 -04004006
Zhao Lei9c170b22015-09-15 21:08:08 +08004007 if (allowed & BTRFS_BLOCK_GROUP_RAID6)
4008 allowed = BTRFS_BLOCK_GROUP_RAID6;
4009 else if (allowed & BTRFS_BLOCK_GROUP_RAID5)
4010 allowed = BTRFS_BLOCK_GROUP_RAID5;
4011 else if (allowed & BTRFS_BLOCK_GROUP_RAID10)
4012 allowed = BTRFS_BLOCK_GROUP_RAID10;
4013 else if (allowed & BTRFS_BLOCK_GROUP_RAID1)
4014 allowed = BTRFS_BLOCK_GROUP_RAID1;
4015 else if (allowed & BTRFS_BLOCK_GROUP_RAID0)
4016 allowed = BTRFS_BLOCK_GROUP_RAID0;
Chris Masonec44a352008-04-28 15:29:52 -04004017
Zhao Lei9c170b22015-09-15 21:08:08 +08004018 flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK;
Chris Masonec44a352008-04-28 15:29:52 -04004019
Zhao Lei9c170b22015-09-15 21:08:08 +08004020 return extended_to_chunk(flags | allowed);
Chris Masonec44a352008-04-28 15:29:52 -04004021}
4022
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004023static u64 get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags)
Josef Bacik6a632092009-02-20 11:00:09 -05004024{
Miao Xiede98ced2013-01-29 10:13:12 +00004025 unsigned seq;
Filipe Mananaf8213bd2014-04-24 15:15:29 +01004026 u64 flags;
Miao Xiede98ced2013-01-29 10:13:12 +00004027
4028 do {
Filipe Mananaf8213bd2014-04-24 15:15:29 +01004029 flags = orig_flags;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004030 seq = read_seqbegin(&fs_info->profiles_lock);
Miao Xiede98ced2013-01-29 10:13:12 +00004031
4032 if (flags & BTRFS_BLOCK_GROUP_DATA)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004033 flags |= fs_info->avail_data_alloc_bits;
Miao Xiede98ced2013-01-29 10:13:12 +00004034 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004035 flags |= fs_info->avail_system_alloc_bits;
Miao Xiede98ced2013-01-29 10:13:12 +00004036 else if (flags & BTRFS_BLOCK_GROUP_METADATA)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004037 flags |= fs_info->avail_metadata_alloc_bits;
4038 } while (read_seqretry(&fs_info->profiles_lock, seq));
Ilya Dryomov6fef8df2012-01-16 22:04:47 +02004039
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004040 return btrfs_reduce_alloc_profile(fs_info, flags);
Yan, Zhengb742bb822010-05-16 10:46:24 -04004041}
Josef Bacik6a632092009-02-20 11:00:09 -05004042
Jeff Mahoney1b868262017-05-17 11:38:35 -04004043static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data)
Yan, Zhengb742bb822010-05-16 10:46:24 -04004044{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004045 struct btrfs_fs_info *fs_info = root->fs_info;
Yan, Zhengb742bb822010-05-16 10:46:24 -04004046 u64 flags;
David Woodhouse53b381b2013-01-29 18:40:14 -05004047 u64 ret;
Josef Bacik6a632092009-02-20 11:00:09 -05004048
Yan, Zhengb742bb822010-05-16 10:46:24 -04004049 if (data)
4050 flags = BTRFS_BLOCK_GROUP_DATA;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004051 else if (root == fs_info->chunk_root)
Yan, Zhengb742bb822010-05-16 10:46:24 -04004052 flags = BTRFS_BLOCK_GROUP_SYSTEM;
4053 else
4054 flags = BTRFS_BLOCK_GROUP_METADATA;
4055
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004056 ret = get_alloc_profile(fs_info, flags);
David Woodhouse53b381b2013-01-29 18:40:14 -05004057 return ret;
Josef Bacik6a632092009-02-20 11:00:09 -05004058}
4059
Jeff Mahoney1b868262017-05-17 11:38:35 -04004060u64 btrfs_data_alloc_profile(struct btrfs_fs_info *fs_info)
4061{
4062 return get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_DATA);
4063}
4064
4065u64 btrfs_metadata_alloc_profile(struct btrfs_fs_info *fs_info)
4066{
4067 return get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4068}
4069
4070u64 btrfs_system_alloc_profile(struct btrfs_fs_info *fs_info)
4071{
4072 return get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
4073}
4074
Liu Bo41361352017-02-13 15:42:21 -08004075static u64 btrfs_space_info_used(struct btrfs_space_info *s_info,
4076 bool may_use_included)
4077{
4078 ASSERT(s_info);
4079 return s_info->bytes_used + s_info->bytes_reserved +
4080 s_info->bytes_pinned + s_info->bytes_readonly +
4081 (may_use_included ? s_info->bytes_may_use : 0);
4082}
4083
Nikolay Borisov04f4f912017-02-20 13:50:36 +02004084int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes)
Josef Bacik6a632092009-02-20 11:00:09 -05004085{
Nikolay Borisov04f4f912017-02-20 13:50:36 +02004086 struct btrfs_root *root = inode->root;
Li Zefanb4d7c3c2012-07-09 20:21:07 -06004087 struct btrfs_fs_info *fs_info = root->fs_info;
Nikolay Borisov1174cad2017-07-11 13:47:50 +03004088 struct btrfs_space_info *data_sinfo = fs_info->data_sinfo;
Josef Bacikab6e24102010-03-19 14:38:13 +00004089 u64 used;
Zhao Lei94b947b2015-02-14 13:23:45 +08004090 int ret = 0;
Zhao Leic99f1b02015-03-02 19:32:20 +08004091 int need_commit = 2;
4092 int have_pinned_space;
Josef Bacik6a632092009-02-20 11:00:09 -05004093
4094 /* make sure bytes are sectorsize aligned */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004095 bytes = ALIGN(bytes, fs_info->sectorsize);
Josef Bacik6a632092009-02-20 11:00:09 -05004096
Miao Xie9dced182013-10-25 17:33:36 +08004097 if (btrfs_is_free_space_inode(inode)) {
Zhao Leic99f1b02015-03-02 19:32:20 +08004098 need_commit = 0;
Miao Xie9dced182013-10-25 17:33:36 +08004099 ASSERT(current->journal_info);
Josef Bacik0af3d002010-06-21 14:48:16 -04004100 }
4101
Josef Bacik6a632092009-02-20 11:00:09 -05004102again:
4103 /* make sure we have enough space to handle the data first */
4104 spin_lock(&data_sinfo->lock);
Liu Bo41361352017-02-13 15:42:21 -08004105 used = btrfs_space_info_used(data_sinfo, true);
Josef Bacikab6e24102010-03-19 14:38:13 +00004106
4107 if (used + bytes > data_sinfo->total_bytes) {
Josef Bacik4e06bdd2009-02-20 10:59:53 -05004108 struct btrfs_trans_handle *trans;
4109
Josef Bacik6a632092009-02-20 11:00:09 -05004110 /*
4111 * if we don't have enough free bytes in this space then we need
4112 * to alloc a new chunk.
4113 */
Zhao Leib9fd47c2015-02-09 14:40:20 +08004114 if (!data_sinfo->full) {
Josef Bacik6a632092009-02-20 11:00:09 -05004115 u64 alloc_target;
Josef Bacik6a632092009-02-20 11:00:09 -05004116
Chris Mason0e4f8f82011-04-15 16:05:44 -04004117 data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
Josef Bacik6a632092009-02-20 11:00:09 -05004118 spin_unlock(&data_sinfo->lock);
Nikolay Borisov1174cad2017-07-11 13:47:50 +03004119
Jeff Mahoney1b868262017-05-17 11:38:35 -04004120 alloc_target = btrfs_data_alloc_profile(fs_info);
Miao Xie9dced182013-10-25 17:33:36 +08004121 /*
4122 * It is ugly that we don't call nolock join
4123 * transaction for the free space inode case here.
4124 * But it is safe because we only do the data space
4125 * reservation for the free space cache in the
4126 * transaction context, the common join transaction
4127 * just increase the counter of the current transaction
4128 * handler, doesn't try to acquire the trans_lock of
4129 * the fs.
4130 */
Josef Bacik7a7eaa42011-04-13 12:54:33 -04004131 trans = btrfs_join_transaction(root);
Yan, Zhenga22285a2010-05-16 10:48:46 -04004132 if (IS_ERR(trans))
4133 return PTR_ERR(trans);
Josef Bacik6a632092009-02-20 11:00:09 -05004134
Nikolay Borisov01458822018-06-20 15:49:05 +03004135 ret = do_chunk_alloc(trans, alloc_target,
Chris Mason0e4f8f82011-04-15 16:05:44 -04004136 CHUNK_ALLOC_NO_FORCE);
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04004137 btrfs_end_transaction(trans);
Miao Xied52a5b52011-01-05 10:07:18 +00004138 if (ret < 0) {
4139 if (ret != -ENOSPC)
4140 return ret;
Zhao Leic99f1b02015-03-02 19:32:20 +08004141 else {
4142 have_pinned_space = 1;
Miao Xied52a5b52011-01-05 10:07:18 +00004143 goto commit_trans;
Zhao Leic99f1b02015-03-02 19:32:20 +08004144 }
Miao Xied52a5b52011-01-05 10:07:18 +00004145 }
Chris Mason33b4d472009-09-22 14:45:50 -04004146
Josef Bacik6a632092009-02-20 11:00:09 -05004147 goto again;
4148 }
Josef Bacikf2bb8f52011-05-25 13:10:16 -04004149
4150 /*
Josef Bacikb150a4f2013-06-19 15:00:04 -04004151 * If we don't have enough pinned space to deal with this
Zhao Lei94b947b2015-02-14 13:23:45 +08004152 * allocation, and no removed chunk in current transaction,
4153 * don't bother committing the transaction.
Josef Bacikf2bb8f52011-05-25 13:10:16 -04004154 */
Ethan Liendec59fa2018-07-13 16:50:42 +08004155 have_pinned_space = __percpu_counter_compare(
Zhao Leic99f1b02015-03-02 19:32:20 +08004156 &data_sinfo->total_bytes_pinned,
Ethan Liendec59fa2018-07-13 16:50:42 +08004157 used + bytes - data_sinfo->total_bytes,
4158 BTRFS_TOTAL_BYTES_PINNED_BATCH);
Josef Bacik6a632092009-02-20 11:00:09 -05004159 spin_unlock(&data_sinfo->lock);
Josef Bacik4e06bdd2009-02-20 10:59:53 -05004160
4161 /* commit the current transaction and try again */
Miao Xied52a5b52011-01-05 10:07:18 +00004162commit_trans:
Nikolay Borisov92e2f7e2018-02-05 10:41:16 +02004163 if (need_commit) {
Zhao Leic99f1b02015-03-02 19:32:20 +08004164 need_commit--;
Josef Bacikb150a4f2013-06-19 15:00:04 -04004165
Zhao Leie1746e82015-12-01 18:39:40 +08004166 if (need_commit > 0) {
Nikolay Borisov82b3e532018-04-23 10:54:13 +03004167 btrfs_start_delalloc_roots(fs_info, -1);
Chris Mason6374e57a2017-06-23 09:48:21 -07004168 btrfs_wait_ordered_roots(fs_info, U64_MAX, 0,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004169 (u64)-1);
Zhao Leie1746e82015-12-01 18:39:40 +08004170 }
Zhao Lei9a4e7272015-04-09 12:34:43 +08004171
Josef Bacik7a7eaa42011-04-13 12:54:33 -04004172 trans = btrfs_join_transaction(root);
Yan, Zhenga22285a2010-05-16 10:48:46 -04004173 if (IS_ERR(trans))
4174 return PTR_ERR(trans);
Zhao Leic99f1b02015-03-02 19:32:20 +08004175 if (have_pinned_space >= 0 ||
Josef Bacik3204d332015-09-24 10:46:10 -04004176 test_bit(BTRFS_TRANS_HAVE_FREE_BGS,
4177 &trans->transaction->flags) ||
Zhao Leic99f1b02015-03-02 19:32:20 +08004178 need_commit > 0) {
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04004179 ret = btrfs_commit_transaction(trans);
Zhao Lei94b947b2015-02-14 13:23:45 +08004180 if (ret)
4181 return ret;
Zhao Leid7c15172015-02-26 10:49:20 +08004182 /*
Filipe Mananac2d6cb12016-01-15 11:05:12 +00004183 * The cleaner kthread might still be doing iput
4184 * operations. Wait for it to finish so that
Josef Bacik034f7842018-12-03 11:06:52 -05004185 * more space is released. We don't need to
4186 * explicitly run the delayed iputs here because
4187 * the commit_transaction would have woken up
4188 * the cleaner.
Zhao Leid7c15172015-02-26 10:49:20 +08004189 */
Josef Bacik034f7842018-12-03 11:06:52 -05004190 ret = btrfs_wait_on_delayed_iputs(fs_info);
4191 if (ret)
4192 return ret;
Zhao Lei94b947b2015-02-14 13:23:45 +08004193 goto again;
4194 } else {
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04004195 btrfs_end_transaction(trans);
Zhao Lei94b947b2015-02-14 13:23:45 +08004196 }
Josef Bacik4e06bdd2009-02-20 10:59:53 -05004197 }
4198
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004199 trace_btrfs_space_reservation(fs_info,
Jeff Mahoneycab45e22013-10-16 16:27:01 -04004200 "space_info:enospc",
4201 data_sinfo->flags, bytes, 1);
Josef Bacik6a632092009-02-20 11:00:09 -05004202 return -ENOSPC;
4203 }
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08004204 update_bytes_may_use(data_sinfo, bytes);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004205 trace_btrfs_space_reservation(fs_info, "space_info",
Liu Bo2bcc0322012-03-29 09:57:44 -04004206 data_sinfo->flags, bytes, 1);
Josef Bacik6a632092009-02-20 11:00:09 -05004207 spin_unlock(&data_sinfo->lock);
4208
Josef Bacik4559b0a2018-07-19 10:49:51 -04004209 return 0;
Josef Bacik6a632092009-02-20 11:00:09 -05004210}
4211
Qu Wenruo364ecf32017-02-27 15:10:38 +08004212int btrfs_check_data_free_space(struct inode *inode,
4213 struct extent_changeset **reserved, u64 start, u64 len)
Qu Wenruo4ceff072015-09-08 17:22:42 +08004214{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004215 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Qu Wenruo4ceff072015-09-08 17:22:42 +08004216 int ret;
4217
4218 /* align the range */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004219 len = round_up(start + len, fs_info->sectorsize) -
4220 round_down(start, fs_info->sectorsize);
4221 start = round_down(start, fs_info->sectorsize);
Qu Wenruo4ceff072015-09-08 17:22:42 +08004222
Nikolay Borisov04f4f912017-02-20 13:50:36 +02004223 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode), len);
Qu Wenruo4ceff072015-09-08 17:22:42 +08004224 if (ret < 0)
4225 return ret;
4226
Josef Bacik1e5ec2e2016-09-15 14:57:48 -04004227 /* Use new btrfs_qgroup_reserve_data to reserve precious data space. */
Qu Wenruo364ecf32017-02-27 15:10:38 +08004228 ret = btrfs_qgroup_reserve_data(inode, reserved, start, len);
Qu Wenruo7bc329c2017-02-27 15:10:36 +08004229 if (ret < 0)
Josef Bacik1e5ec2e2016-09-15 14:57:48 -04004230 btrfs_free_reserved_data_space_noquota(inode, start, len);
Qu Wenruo364ecf32017-02-27 15:10:38 +08004231 else
4232 ret = 0;
Qu Wenruo4ceff072015-09-08 17:22:42 +08004233 return ret;
4234}
4235
4236/*
Qu Wenruo4ceff072015-09-08 17:22:42 +08004237 * Called if we need to clear a data reservation for this inode
4238 * Normally in a error case.
4239 *
Qu Wenruo51773be2015-10-08 18:19:37 +08004240 * This one will *NOT* use accurate qgroup reserved space API, just for case
4241 * which we can't sleep and is sure it won't affect qgroup reserved space.
4242 * Like clear_bit_hook().
Qu Wenruo4ceff072015-09-08 17:22:42 +08004243 */
Qu Wenruo51773be2015-10-08 18:19:37 +08004244void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start,
4245 u64 len)
Qu Wenruo4ceff072015-09-08 17:22:42 +08004246{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004247 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Qu Wenruo4ceff072015-09-08 17:22:42 +08004248 struct btrfs_space_info *data_sinfo;
4249
4250 /* Make sure the range is aligned to sectorsize */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004251 len = round_up(start + len, fs_info->sectorsize) -
4252 round_down(start, fs_info->sectorsize);
4253 start = round_down(start, fs_info->sectorsize);
Qu Wenruo4ceff072015-09-08 17:22:42 +08004254
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004255 data_sinfo = fs_info->data_sinfo;
Qu Wenruo4ceff072015-09-08 17:22:42 +08004256 spin_lock(&data_sinfo->lock);
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08004257 update_bytes_may_use(data_sinfo, -len);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004258 trace_btrfs_space_reservation(fs_info, "space_info",
Qu Wenruo4ceff072015-09-08 17:22:42 +08004259 data_sinfo->flags, len, 0);
4260 spin_unlock(&data_sinfo->lock);
4261}
4262
Qu Wenruo51773be2015-10-08 18:19:37 +08004263/*
4264 * Called if we need to clear a data reservation for this inode
4265 * Normally in a error case.
4266 *
Nicholas D Steeves01327612016-05-19 21:18:45 -04004267 * This one will handle the per-inode data rsv map for accurate reserved
Qu Wenruo51773be2015-10-08 18:19:37 +08004268 * space framework.
4269 */
Qu Wenruobc42bda2017-02-27 15:10:39 +08004270void btrfs_free_reserved_data_space(struct inode *inode,
4271 struct extent_changeset *reserved, u64 start, u64 len)
Qu Wenruo51773be2015-10-08 18:19:37 +08004272{
Jeff Mahoney0c476a52016-11-18 21:52:40 -05004273 struct btrfs_root *root = BTRFS_I(inode)->root;
4274
4275 /* Make sure the range is aligned to sectorsize */
Jeff Mahoneyda170662016-06-15 09:22:56 -04004276 len = round_up(start + len, root->fs_info->sectorsize) -
4277 round_down(start, root->fs_info->sectorsize);
4278 start = round_down(start, root->fs_info->sectorsize);
Jeff Mahoney0c476a52016-11-18 21:52:40 -05004279
Qu Wenruo51773be2015-10-08 18:19:37 +08004280 btrfs_free_reserved_data_space_noquota(inode, start, len);
Qu Wenruobc42bda2017-02-27 15:10:39 +08004281 btrfs_qgroup_free_data(inode, reserved, start, len);
Qu Wenruo51773be2015-10-08 18:19:37 +08004282}
4283
Josef Bacik97e728d2009-04-21 17:40:57 -04004284static void force_metadata_allocation(struct btrfs_fs_info *info)
4285{
4286 struct list_head *head = &info->space_info;
4287 struct btrfs_space_info *found;
4288
4289 rcu_read_lock();
4290 list_for_each_entry_rcu(found, head, list) {
4291 if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
Chris Mason0e4f8f82011-04-15 16:05:44 -04004292 found->force_alloc = CHUNK_ALLOC_FORCE;
Josef Bacik97e728d2009-04-21 17:40:57 -04004293 }
4294 rcu_read_unlock();
4295}
4296
Miao Xie3c76cd82013-04-25 10:12:38 +00004297static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global)
4298{
4299 return (global->size << 1);
4300}
4301
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004302static int should_alloc_chunk(struct btrfs_fs_info *fs_info,
Josef Bacik698d0082012-09-12 14:08:47 -04004303 struct btrfs_space_info *sinfo, int force)
Yan, Zheng424499d2010-05-16 10:46:25 -04004304{
Nikolay Borisov8d8aafe2017-06-22 09:51:48 -04004305 u64 bytes_used = btrfs_space_info_used(sinfo, false);
Chris Masone5bc2452010-10-26 13:37:56 -04004306 u64 thresh;
Yan, Zheng424499d2010-05-16 10:46:25 -04004307
Chris Mason0e4f8f82011-04-15 16:05:44 -04004308 if (force == CHUNK_ALLOC_FORCE)
4309 return 1;
4310
4311 /*
4312 * in limited mode, we want to have some free space up to
4313 * about 1% of the FS size.
4314 */
4315 if (force == CHUNK_ALLOC_LIMITED) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004316 thresh = btrfs_super_total_bytes(fs_info->super_copy);
Byongho Leeee221842015-12-15 01:42:10 +09004317 thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1));
Chris Mason0e4f8f82011-04-15 16:05:44 -04004318
Nikolay Borisov8d8aafe2017-06-22 09:51:48 -04004319 if (sinfo->total_bytes - bytes_used < thresh)
Chris Mason0e4f8f82011-04-15 16:05:44 -04004320 return 1;
4321 }
Chris Mason0e4f8f82011-04-15 16:05:44 -04004322
Nikolay Borisov8d8aafe2017-06-22 09:51:48 -04004323 if (bytes_used + SZ_2M < div_factor(sinfo->total_bytes, 8))
Josef Bacik14ed0ca2010-10-15 15:23:48 -04004324 return 0;
Yan, Zheng424499d2010-05-16 10:46:25 -04004325 return 1;
4326}
4327
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004328static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type)
Liu Bo15d1ff82012-03-29 09:57:44 -04004329{
4330 u64 num_dev;
4331
David Woodhouse53b381b2013-01-29 18:40:14 -05004332 if (type & (BTRFS_BLOCK_GROUP_RAID10 |
4333 BTRFS_BLOCK_GROUP_RAID0 |
4334 BTRFS_BLOCK_GROUP_RAID5 |
4335 BTRFS_BLOCK_GROUP_RAID6))
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004336 num_dev = fs_info->fs_devices->rw_devices;
Liu Bo15d1ff82012-03-29 09:57:44 -04004337 else if (type & BTRFS_BLOCK_GROUP_RAID1)
4338 num_dev = 2;
4339 else
4340 num_dev = 1; /* DUP or single */
4341
Filipe Manana39c2d7f2015-05-20 14:01:55 +01004342 return num_dev;
Liu Bo15d1ff82012-03-29 09:57:44 -04004343}
4344
Filipe Manana39c2d7f2015-05-20 14:01:55 +01004345/*
4346 * If @is_allocation is true, reserve space in the system space info necessary
4347 * for allocating a chunk, otherwise if it's false, reserve space necessary for
4348 * removing a chunk.
4349 */
Nikolay Borisov451a2c12018-06-20 15:49:07 +03004350void check_system_chunk(struct btrfs_trans_handle *trans, u64 type)
Liu Bo15d1ff82012-03-29 09:57:44 -04004351{
Nikolay Borisov451a2c12018-06-20 15:49:07 +03004352 struct btrfs_fs_info *fs_info = trans->fs_info;
Liu Bo15d1ff82012-03-29 09:57:44 -04004353 struct btrfs_space_info *info;
4354 u64 left;
4355 u64 thresh;
Filipe Manana4fbcdf62015-05-20 14:01:54 +01004356 int ret = 0;
Filipe Manana39c2d7f2015-05-20 14:01:55 +01004357 u64 num_devs;
Filipe Manana4fbcdf62015-05-20 14:01:54 +01004358
4359 /*
4360 * Needed because we can end up allocating a system chunk and for an
4361 * atomic and race free space reservation in the chunk block reserve.
4362 */
David Sterbaa32bf9a2018-03-16 02:21:22 +01004363 lockdep_assert_held(&fs_info->chunk_mutex);
Liu Bo15d1ff82012-03-29 09:57:44 -04004364
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004365 info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
Liu Bo15d1ff82012-03-29 09:57:44 -04004366 spin_lock(&info->lock);
Liu Bo41361352017-02-13 15:42:21 -08004367 left = info->total_bytes - btrfs_space_info_used(info, true);
Liu Bo15d1ff82012-03-29 09:57:44 -04004368 spin_unlock(&info->lock);
4369
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004370 num_devs = get_profile_num_devs(fs_info, type);
Filipe Manana39c2d7f2015-05-20 14:01:55 +01004371
4372 /* num_devs device items to update and 1 chunk item to add or remove */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004373 thresh = btrfs_calc_trunc_metadata_size(fs_info, num_devs) +
4374 btrfs_calc_trans_metadata_size(fs_info, 1);
Filipe Manana39c2d7f2015-05-20 14:01:55 +01004375
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004376 if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
4377 btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu",
4378 left, thresh, type);
4379 dump_space_info(fs_info, info, 0, 0);
Liu Bo15d1ff82012-03-29 09:57:44 -04004380 }
4381
4382 if (left < thresh) {
Jeff Mahoney1b868262017-05-17 11:38:35 -04004383 u64 flags = btrfs_system_alloc_profile(fs_info);
Liu Bo15d1ff82012-03-29 09:57:44 -04004384
Filipe Manana4fbcdf62015-05-20 14:01:54 +01004385 /*
4386 * Ignore failure to create system chunk. We might end up not
4387 * needing it, as we might not need to COW all nodes/leafs from
4388 * the paths we visit in the chunk tree (they were already COWed
4389 * or created in the current transaction for example).
4390 */
Nikolay Borisovc216b202018-06-20 15:49:06 +03004391 ret = btrfs_alloc_chunk(trans, flags);
Filipe Manana4fbcdf62015-05-20 14:01:54 +01004392 }
4393
4394 if (!ret) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004395 ret = btrfs_block_rsv_add(fs_info->chunk_root,
4396 &fs_info->chunk_block_rsv,
Filipe Manana4fbcdf62015-05-20 14:01:54 +01004397 thresh, BTRFS_RESERVE_NO_FLUSH);
4398 if (!ret)
4399 trans->chunk_bytes_reserved += thresh;
Liu Bo15d1ff82012-03-29 09:57:44 -04004400 }
4401}
4402
Liu Bo28b737f2016-07-29 11:09:50 -07004403/*
4404 * If force is CHUNK_ALLOC_FORCE:
4405 * - return 1 if it successfully allocates a chunk,
4406 * - return errors including -ENOSPC otherwise.
4407 * If force is NOT CHUNK_ALLOC_FORCE:
4408 * - return 0 if it doesn't need to allocate a new chunk,
4409 * - return 1 if it successfully allocates a chunk,
4410 * - return errors including -ENOSPC otherwise.
4411 */
Nikolay Borisov01458822018-06-20 15:49:05 +03004412static int do_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags,
4413 int force)
Chris Mason6324fbf2008-03-24 15:01:59 -04004414{
Nikolay Borisov01458822018-06-20 15:49:05 +03004415 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Mason6324fbf2008-03-24 15:01:59 -04004416 struct btrfs_space_info *space_info;
Nikolay Borisov2556fbb2018-04-18 10:27:57 +03004417 bool wait_for_alloc = false;
4418 bool should_alloc = false;
Yan Zhengc146afa2008-11-12 14:34:12 -05004419 int ret = 0;
4420
Josef Bacikc6b305a2012-12-18 09:16:16 -05004421 /* Don't re-enter if we're already allocating a chunk */
4422 if (trans->allocating_chunk)
4423 return -ENOSPC;
4424
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004425 space_info = __find_space_info(fs_info, flags);
Jeff Mahoneydc2d3002018-03-20 15:25:25 -04004426 ASSERT(space_info);
Chris Mason6324fbf2008-03-24 15:01:59 -04004427
Nikolay Borisov2556fbb2018-04-18 10:27:57 +03004428 do {
4429 spin_lock(&space_info->lock);
4430 if (force < space_info->force_alloc)
4431 force = space_info->force_alloc;
4432 should_alloc = should_alloc_chunk(fs_info, space_info, force);
4433 if (space_info->full) {
4434 /* No more free physical space */
4435 if (should_alloc)
4436 ret = -ENOSPC;
4437 else
4438 ret = 0;
4439 spin_unlock(&space_info->lock);
4440 return ret;
4441 } else if (!should_alloc) {
4442 spin_unlock(&space_info->lock);
4443 return 0;
4444 } else if (space_info->chunk_alloc) {
4445 /*
4446 * Someone is already allocating, so we need to block
4447 * until this someone is finished and then loop to
4448 * recheck if we should continue with our allocation
4449 * attempt.
4450 */
4451 wait_for_alloc = true;
4452 spin_unlock(&space_info->lock);
4453 mutex_lock(&fs_info->chunk_mutex);
4454 mutex_unlock(&fs_info->chunk_mutex);
4455 } else {
4456 /* Proceed with allocation */
4457 space_info->chunk_alloc = 1;
4458 wait_for_alloc = false;
4459 spin_unlock(&space_info->lock);
4460 }
Chris Mason6324fbf2008-03-24 15:01:59 -04004461
Nikolay Borisov2556fbb2018-04-18 10:27:57 +03004462 cond_resched();
4463 } while (wait_for_alloc);
Josef Bacik25179202008-10-29 14:49:05 -04004464
Josef Bacik6d741192011-04-11 20:20:11 -04004465 mutex_lock(&fs_info->chunk_mutex);
Josef Bacikc6b305a2012-12-18 09:16:16 -05004466 trans->allocating_chunk = true;
4467
Josef Bacik97e728d2009-04-21 17:40:57 -04004468 /*
Josef Bacik67377732010-09-16 16:19:09 -04004469 * If we have mixed data/metadata chunks we want to make sure we keep
4470 * allocating mixed chunks instead of individual chunks.
4471 */
4472 if (btrfs_mixed_space_info(space_info))
4473 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
4474
4475 /*
Josef Bacik97e728d2009-04-21 17:40:57 -04004476 * if we're doing a data chunk, go ahead and make sure that
4477 * we keep a reasonable number of metadata chunks allocated in the
4478 * FS as well.
4479 */
Josef Bacik9ed74f22009-09-11 16:12:44 -04004480 if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
Josef Bacik97e728d2009-04-21 17:40:57 -04004481 fs_info->data_chunk_allocations++;
4482 if (!(fs_info->data_chunk_allocations %
4483 fs_info->metadata_ratio))
4484 force_metadata_allocation(fs_info);
4485 }
4486
Liu Bo15d1ff82012-03-29 09:57:44 -04004487 /*
4488 * Check if we have enough space in SYSTEM chunk because we may need
4489 * to update devices.
4490 */
Nikolay Borisov451a2c12018-06-20 15:49:07 +03004491 check_system_chunk(trans, flags);
Liu Bo15d1ff82012-03-29 09:57:44 -04004492
Nikolay Borisovc216b202018-06-20 15:49:06 +03004493 ret = btrfs_alloc_chunk(trans, flags);
Josef Bacikc6b305a2012-12-18 09:16:16 -05004494 trans->allocating_chunk = false;
Mark Fasheh92b8e8972011-07-12 10:57:59 -07004495
Josef Bacik9ed74f22009-09-11 16:12:44 -04004496 spin_lock(&space_info->lock);
Nikolay Borisov57f16422018-04-11 11:21:19 +03004497 if (ret < 0) {
4498 if (ret == -ENOSPC)
4499 space_info->full = 1;
4500 else
4501 goto out;
4502 } else {
Yan, Zheng424499d2010-05-16 10:46:25 -04004503 ret = 1;
Josef Bacik21a94f72018-10-11 15:54:03 -04004504 space_info->max_extent_size = 0;
Nikolay Borisov57f16422018-04-11 11:21:19 +03004505 }
Josef Bacik6d741192011-04-11 20:20:11 -04004506
Chris Mason0e4f8f82011-04-15 16:05:44 -04004507 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
Alexandre Olivaa81cb9a2013-02-21 21:15:14 +00004508out:
Josef Bacik6d741192011-04-11 20:20:11 -04004509 space_info->chunk_alloc = 0;
Josef Bacik9ed74f22009-09-11 16:12:44 -04004510 spin_unlock(&space_info->lock);
Dan Carpentera25c75d2012-04-18 09:59:29 +03004511 mutex_unlock(&fs_info->chunk_mutex);
Filipe Manana00d80e32015-07-20 14:56:20 +01004512 /*
4513 * When we allocate a new chunk we reserve space in the chunk block
4514 * reserve to make sure we can COW nodes/leafs in the chunk tree or
4515 * add new nodes/leafs to it if we end up needing to do it when
4516 * inserting the chunk item and updating device items as part of the
4517 * second phase of chunk allocation, performed by
4518 * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a
4519 * large number of new block groups to create in our transaction
4520 * handle's new_bgs list to avoid exhausting the chunk block reserve
4521 * in extreme cases - like having a single transaction create many new
4522 * block groups when starting to write out the free space caches of all
4523 * the block groups that were made dirty during the lifetime of the
4524 * transaction.
4525 */
Filipe Manana5ce55552018-10-12 10:03:55 +01004526 if (trans->chunk_bytes_reserved >= (u64)SZ_2M)
Nikolay Borisov6c686b32018-02-07 17:55:40 +02004527 btrfs_create_pending_block_groups(trans);
Filipe Manana5ce55552018-10-12 10:03:55 +01004528
Josef Bacik0f9dd462008-09-23 13:14:11 -04004529 return ret;
Chris Mason6324fbf2008-03-24 15:01:59 -04004530}
4531
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004532static int can_overcommit(struct btrfs_fs_info *fs_info,
Josef Bacika80c8dc2012-09-06 16:59:33 -04004533 struct btrfs_space_info *space_info, u64 bytes,
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004534 enum btrfs_reserve_flush_enum flush,
4535 bool system_chunk)
Josef Bacika80c8dc2012-09-06 16:59:33 -04004536{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004537 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
Josef Bacik957780e2016-05-17 13:30:55 -04004538 u64 profile;
Miao Xie3c76cd82013-04-25 10:12:38 +00004539 u64 space_size;
Josef Bacika80c8dc2012-09-06 16:59:33 -04004540 u64 avail;
4541 u64 used;
David Sterba46df06b2018-07-13 20:46:30 +02004542 int factor;
Josef Bacika80c8dc2012-09-06 16:59:33 -04004543
Josef Bacik957780e2016-05-17 13:30:55 -04004544 /* Don't overcommit when in mixed mode. */
4545 if (space_info->flags & BTRFS_BLOCK_GROUP_DATA)
4546 return 0;
4547
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004548 if (system_chunk)
4549 profile = btrfs_system_alloc_profile(fs_info);
4550 else
4551 profile = btrfs_metadata_alloc_profile(fs_info);
4552
Liu Bo41361352017-02-13 15:42:21 -08004553 used = btrfs_space_info_used(space_info, false);
Josef Bacik96f1bb52013-01-30 17:02:51 -05004554
Josef Bacik96f1bb52013-01-30 17:02:51 -05004555 /*
4556 * We only want to allow over committing if we have lots of actual space
4557 * free, but if we don't have enough space to handle the global reserve
4558 * space then we could end up having a real enospc problem when trying
4559 * to allocate a chunk or some other such important allocation.
4560 */
Miao Xie3c76cd82013-04-25 10:12:38 +00004561 spin_lock(&global_rsv->lock);
4562 space_size = calc_global_rsv_need_space(global_rsv);
4563 spin_unlock(&global_rsv->lock);
4564 if (used + space_size >= space_info->total_bytes)
Josef Bacik96f1bb52013-01-30 17:02:51 -05004565 return 0;
4566
4567 used += space_info->bytes_may_use;
Josef Bacika80c8dc2012-09-06 16:59:33 -04004568
Nikolay Borisova5ed45f2017-05-11 09:17:46 +03004569 avail = atomic64_read(&fs_info->free_chunk_space);
Josef Bacika80c8dc2012-09-06 16:59:33 -04004570
4571 /*
4572 * If we have dup, raid1 or raid10 then only half of the free
Andrea Gelmini52042d82018-11-28 12:05:13 +01004573 * space is actually usable. For raid56, the space info used
David Woodhouse53b381b2013-01-29 18:40:14 -05004574 * doesn't include the parity drive, so we don't have to
4575 * change the math
Josef Bacika80c8dc2012-09-06 16:59:33 -04004576 */
David Sterba46df06b2018-07-13 20:46:30 +02004577 factor = btrfs_bg_type_to_factor(profile);
4578 avail = div_u64(avail, factor);
Josef Bacika80c8dc2012-09-06 16:59:33 -04004579
4580 /*
Miao Xie561c2942012-10-16 11:32:18 +00004581 * If we aren't flushing all things, let us overcommit up to
4582 * 1/2th of the space. If we can flush, don't let us overcommit
4583 * too much, let it overcommit up to 1/8 of the space.
Josef Bacika80c8dc2012-09-06 16:59:33 -04004584 */
Miao Xie08e007d2012-10-16 11:33:38 +00004585 if (flush == BTRFS_RESERVE_FLUSH_ALL)
Josef Bacik14575ae2013-09-17 10:48:00 -04004586 avail >>= 3;
Josef Bacika80c8dc2012-09-06 16:59:33 -04004587 else
Josef Bacik14575ae2013-09-17 10:48:00 -04004588 avail >>= 1;
Josef Bacika80c8dc2012-09-06 16:59:33 -04004589
Josef Bacik14575ae2013-09-17 10:48:00 -04004590 if (used + bytes < space_info->total_bytes + avail)
Josef Bacika80c8dc2012-09-06 16:59:33 -04004591 return 1;
4592 return 0;
4593}
4594
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004595static void btrfs_writeback_inodes_sb_nr(struct btrfs_fs_info *fs_info,
Miao Xie6c255e62014-03-06 13:55:01 +08004596 unsigned long nr_pages, int nr_items)
Miao Xieda633a42012-12-20 11:19:09 +00004597{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004598 struct super_block *sb = fs_info->sb;
Miao Xieda633a42012-12-20 11:19:09 +00004599
Josef Bacik925a6ef2013-06-20 12:31:27 -04004600 if (down_read_trylock(&sb->s_umount)) {
4601 writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE);
4602 up_read(&sb->s_umount);
4603 } else {
Miao Xieda633a42012-12-20 11:19:09 +00004604 /*
4605 * We needn't worry the filesystem going from r/w to r/o though
4606 * we don't acquire ->s_umount mutex, because the filesystem
4607 * should guarantee the delalloc inodes list be empty after
4608 * the filesystem is readonly(all dirty pages are written to
4609 * the disk).
4610 */
Nikolay Borisov82b3e532018-04-23 10:54:13 +03004611 btrfs_start_delalloc_roots(fs_info, nr_items);
Josef Bacik98ad69c2013-04-04 11:55:49 -04004612 if (!current->journal_info)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004613 btrfs_wait_ordered_roots(fs_info, nr_items, 0, (u64)-1);
Miao Xieda633a42012-12-20 11:19:09 +00004614 }
4615}
4616
Chris Mason6374e57a2017-06-23 09:48:21 -07004617static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004618 u64 to_reclaim)
Miao Xie18cd8ea2013-11-04 23:13:22 +08004619{
4620 u64 bytes;
Chris Mason6374e57a2017-06-23 09:48:21 -07004621 u64 nr;
Miao Xie18cd8ea2013-11-04 23:13:22 +08004622
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004623 bytes = btrfs_calc_trans_metadata_size(fs_info, 1);
Chris Mason6374e57a2017-06-23 09:48:21 -07004624 nr = div64_u64(to_reclaim, bytes);
Miao Xie18cd8ea2013-11-04 23:13:22 +08004625 if (!nr)
4626 nr = 1;
4627 return nr;
4628}
4629
Byongho Leeee221842015-12-15 01:42:10 +09004630#define EXTENT_SIZE_PER_ITEM SZ_256K
Miao Xiec61a16a2013-11-04 23:13:23 +08004631
Yan, Zheng5da9d012010-05-16 10:46:25 -04004632/*
4633 * shrink metadata reservation for delalloc
4634 */
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004635static void shrink_delalloc(struct btrfs_fs_info *fs_info, u64 to_reclaim,
4636 u64 orig, bool wait_ordered)
Yan, Zheng5da9d012010-05-16 10:46:25 -04004637{
Josef Bacik0019f102010-10-15 15:18:40 -04004638 struct btrfs_space_info *space_info;
Josef Bacik663350a2011-11-03 22:54:25 -04004639 struct btrfs_trans_handle *trans;
Josef Bacikf4c738c2012-07-02 17:10:51 -04004640 u64 delalloc_bytes;
Nikolay Borisov420829d2019-01-03 10:50:05 +02004641 u64 async_pages;
Chris Mason6374e57a2017-06-23 09:48:21 -07004642 u64 items;
Josef Bacikb1953bc2011-01-21 21:10:01 +00004643 long time_left;
Miao Xied3ee29e32013-11-04 23:13:20 +08004644 unsigned long nr_pages;
4645 int loops;
Yan, Zheng5da9d012010-05-16 10:46:25 -04004646
Miao Xiec61a16a2013-11-04 23:13:23 +08004647 /* Calc the number of the pages we need flush for space reservation */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004648 items = calc_reclaim_items_nr(fs_info, to_reclaim);
Chris Mason6374e57a2017-06-23 09:48:21 -07004649 to_reclaim = items * EXTENT_SIZE_PER_ITEM;
Miao Xiec61a16a2013-11-04 23:13:23 +08004650
Josef Bacik663350a2011-11-03 22:54:25 -04004651 trans = (struct btrfs_trans_handle *)current->journal_info;
Josef Bacik69fe2d72017-10-19 14:15:57 -04004652 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
Chris Masonbf9022e2010-10-26 13:40:45 -04004653
Miao Xie963d6782013-01-29 10:10:51 +00004654 delalloc_bytes = percpu_counter_sum_positive(
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004655 &fs_info->delalloc_bytes);
Josef Bacikf4c738c2012-07-02 17:10:51 -04004656 if (delalloc_bytes == 0) {
Josef Bacikfdb5eff2011-06-07 16:07:44 -04004657 if (trans)
Josef Bacikf4c738c2012-07-02 17:10:51 -04004658 return;
Miao Xie38c135a2013-11-04 23:13:21 +08004659 if (wait_ordered)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004660 btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1);
Josef Bacikf4c738c2012-07-02 17:10:51 -04004661 return;
Josef Bacikfdb5eff2011-06-07 16:07:44 -04004662 }
4663
Miao Xied3ee29e32013-11-04 23:13:20 +08004664 loops = 0;
Josef Bacikf4c738c2012-07-02 17:10:51 -04004665 while (delalloc_bytes && loops < 3) {
Nikolay Borisov420829d2019-01-03 10:50:05 +02004666 nr_pages = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT;
4667
Josef Bacikdea31f52012-09-06 16:47:00 -04004668 /*
Nikolay Borisov420829d2019-01-03 10:50:05 +02004669 * Triggers inode writeback for up to nr_pages. This will invoke
4670 * ->writepages callback and trigger delalloc filling
4671 * (btrfs_run_delalloc_range()).
Josef Bacikdea31f52012-09-06 16:47:00 -04004672 */
Nikolay Borisov420829d2019-01-03 10:50:05 +02004673 btrfs_writeback_inodes_sb_nr(fs_info, nr_pages, items);
4674
4675 /*
4676 * We need to wait for the compressed pages to start before
4677 * we continue.
4678 */
4679 async_pages = atomic_read(&fs_info->async_delalloc_pages);
4680 if (!async_pages)
Miao Xie9f3a0742013-11-04 23:13:24 +08004681 goto skip_async;
Josef Bacikdea31f52012-09-06 16:47:00 -04004682
Nikolay Borisov420829d2019-01-03 10:50:05 +02004683 /*
4684 * Calculate how many compressed pages we want to be written
4685 * before we continue. I.e if there are more async pages than we
4686 * require wait_event will wait until nr_pages are written.
4687 */
4688 if (async_pages <= nr_pages)
4689 async_pages = 0;
Miao Xie9f3a0742013-11-04 23:13:24 +08004690 else
Nikolay Borisov420829d2019-01-03 10:50:05 +02004691 async_pages -= nr_pages;
Miao Xie9f3a0742013-11-04 23:13:24 +08004692
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004693 wait_event(fs_info->async_submit_wait,
4694 atomic_read(&fs_info->async_delalloc_pages) <=
Nikolay Borisov420829d2019-01-03 10:50:05 +02004695 (int)async_pages);
Miao Xie9f3a0742013-11-04 23:13:24 +08004696skip_async:
Josef Bacik0019f102010-10-15 15:18:40 -04004697 spin_lock(&space_info->lock);
Josef Bacik957780e2016-05-17 13:30:55 -04004698 if (list_empty(&space_info->tickets) &&
4699 list_empty(&space_info->priority_tickets)) {
4700 spin_unlock(&space_info->lock);
4701 break;
4702 }
Josef Bacik0019f102010-10-15 15:18:40 -04004703 spin_unlock(&space_info->lock);
Yan, Zheng5da9d012010-05-16 10:46:25 -04004704
Chris Mason36e39c42011-03-12 07:08:42 -05004705 loops++;
Josef Bacikf104d042011-10-14 13:56:58 -04004706 if (wait_ordered && !trans) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004707 btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1);
Josef Bacikf104d042011-10-14 13:56:58 -04004708 } else {
Josef Bacikf4c738c2012-07-02 17:10:51 -04004709 time_left = schedule_timeout_killable(1);
Josef Bacikf104d042011-10-14 13:56:58 -04004710 if (time_left)
4711 break;
4712 }
Miao Xie963d6782013-01-29 10:10:51 +00004713 delalloc_bytes = percpu_counter_sum_positive(
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004714 &fs_info->delalloc_bytes);
Yan, Zheng5da9d012010-05-16 10:46:25 -04004715 }
Yan, Zheng5da9d012010-05-16 10:46:25 -04004716}
4717
Josef Bacik996478c2017-08-22 16:00:39 -04004718struct reserve_ticket {
Josef Bacikf91587e2018-11-21 14:03:10 -05004719 u64 orig_bytes;
Josef Bacik996478c2017-08-22 16:00:39 -04004720 u64 bytes;
4721 int error;
4722 struct list_head list;
4723 wait_queue_head_t wait;
4724};
4725
Josef Bacik4a92b1b2011-08-30 12:34:28 -04004726/**
Josef Bacik663350a2011-11-03 22:54:25 -04004727 * maybe_commit_transaction - possibly commit the transaction if its ok to
4728 * @root - the root we're allocating for
4729 * @bytes - the number of bytes we want to reserve
4730 * @force - force the commit
Josef Bacik8bb8ab22010-10-15 16:52:49 -04004731 *
Josef Bacik663350a2011-11-03 22:54:25 -04004732 * This will check to make sure that committing the transaction will actually
4733 * get us somewhere and then commit the transaction if it does. Otherwise it
4734 * will return -ENOSPC.
Josef Bacik8bb8ab22010-10-15 16:52:49 -04004735 */
Jeff Mahoney0c9ab342017-02-15 16:28:28 -05004736static int may_commit_transaction(struct btrfs_fs_info *fs_info,
Josef Bacik996478c2017-08-22 16:00:39 -04004737 struct btrfs_space_info *space_info)
Josef Bacik663350a2011-11-03 22:54:25 -04004738{
Josef Bacik996478c2017-08-22 16:00:39 -04004739 struct reserve_ticket *ticket = NULL;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004740 struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv;
Josef Bacik4c8edbc2018-12-03 10:20:34 -05004741 struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv;
Josef Bacik663350a2011-11-03 22:54:25 -04004742 struct btrfs_trans_handle *trans;
Josef Bacik4c8edbc2018-12-03 10:20:34 -05004743 u64 bytes_needed;
4744 u64 reclaim_bytes = 0;
Josef Bacik663350a2011-11-03 22:54:25 -04004745
4746 trans = (struct btrfs_trans_handle *)current->journal_info;
4747 if (trans)
4748 return -EAGAIN;
4749
Josef Bacik996478c2017-08-22 16:00:39 -04004750 spin_lock(&space_info->lock);
4751 if (!list_empty(&space_info->priority_tickets))
4752 ticket = list_first_entry(&space_info->priority_tickets,
4753 struct reserve_ticket, list);
4754 else if (!list_empty(&space_info->tickets))
4755 ticket = list_first_entry(&space_info->tickets,
4756 struct reserve_ticket, list);
Josef Bacik4c8edbc2018-12-03 10:20:34 -05004757 bytes_needed = (ticket) ? ticket->bytes : 0;
Josef Bacik996478c2017-08-22 16:00:39 -04004758 spin_unlock(&space_info->lock);
4759
Josef Bacik4c8edbc2018-12-03 10:20:34 -05004760 if (!bytes_needed)
Josef Bacik996478c2017-08-22 16:00:39 -04004761 return 0;
Josef Bacik663350a2011-11-03 22:54:25 -04004762
Josef Bacikd89dbef2018-11-21 14:03:06 -05004763 trans = btrfs_join_transaction(fs_info->extent_root);
4764 if (IS_ERR(trans))
4765 return PTR_ERR(trans);
4766
4767 /*
4768 * See if there is enough pinned space to make this reservation, or if
4769 * we have block groups that are going to be freed, allowing us to
4770 * possibly do a chunk allocation the next loop through.
4771 */
4772 if (test_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags) ||
4773 __percpu_counter_compare(&space_info->total_bytes_pinned,
4774 bytes_needed,
4775 BTRFS_TOTAL_BYTES_PINNED_BATCH) >= 0)
Josef Bacik663350a2011-11-03 22:54:25 -04004776 goto commit;
Josef Bacik663350a2011-11-03 22:54:25 -04004777
4778 /*
4779 * See if there is some space in the delayed insertion reservation for
4780 * this reservation.
4781 */
4782 if (space_info != delayed_rsv->space_info)
Josef Bacikd89dbef2018-11-21 14:03:06 -05004783 goto enospc;
Josef Bacik663350a2011-11-03 22:54:25 -04004784
4785 spin_lock(&delayed_rsv->lock);
Josef Bacik4c8edbc2018-12-03 10:20:34 -05004786 reclaim_bytes += delayed_rsv->reserved;
Nikolay Borisov057aac32017-11-07 11:22:54 +02004787 spin_unlock(&delayed_rsv->lock);
4788
Josef Bacik4c8edbc2018-12-03 10:20:34 -05004789 spin_lock(&delayed_refs_rsv->lock);
4790 reclaim_bytes += delayed_refs_rsv->reserved;
4791 spin_unlock(&delayed_refs_rsv->lock);
4792 if (reclaim_bytes >= bytes_needed)
4793 goto commit;
4794 bytes_needed -= reclaim_bytes;
4795
Ethan Liendec59fa2018-07-13 16:50:42 +08004796 if (__percpu_counter_compare(&space_info->total_bytes_pinned,
Josef Bacik4c8edbc2018-12-03 10:20:34 -05004797 bytes_needed,
Josef Bacikd89dbef2018-11-21 14:03:06 -05004798 BTRFS_TOTAL_BYTES_PINNED_BATCH) < 0)
4799 goto enospc;
Josef Bacik663350a2011-11-03 22:54:25 -04004800
4801commit:
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04004802 return btrfs_commit_transaction(trans);
Josef Bacikd89dbef2018-11-21 14:03:06 -05004803enospc:
4804 btrfs_end_transaction(trans);
4805 return -ENOSPC;
Josef Bacik663350a2011-11-03 22:54:25 -04004806}
4807
Nikolay Borisove38ae7a2017-07-25 17:48:28 +03004808/*
4809 * Try to flush some data based on policy set by @state. This is only advisory
4810 * and may fail for various reasons. The caller is supposed to examine the
4811 * state of @space_info to detect the outcome.
4812 */
4813static void flush_space(struct btrfs_fs_info *fs_info,
Josef Bacik96c3f432012-06-21 14:05:49 -04004814 struct btrfs_space_info *space_info, u64 num_bytes,
Nikolay Borisov7bdd6272017-07-11 13:25:13 +03004815 int state)
Josef Bacik96c3f432012-06-21 14:05:49 -04004816{
Jeff Mahoneya9b33112017-05-17 11:38:34 -04004817 struct btrfs_root *root = fs_info->extent_root;
Josef Bacik96c3f432012-06-21 14:05:49 -04004818 struct btrfs_trans_handle *trans;
4819 int nr;
Josef Bacikf4c738c2012-07-02 17:10:51 -04004820 int ret = 0;
Josef Bacik96c3f432012-06-21 14:05:49 -04004821
4822 switch (state) {
Josef Bacik96c3f432012-06-21 14:05:49 -04004823 case FLUSH_DELAYED_ITEMS_NR:
4824 case FLUSH_DELAYED_ITEMS:
Miao Xie18cd8ea2013-11-04 23:13:22 +08004825 if (state == FLUSH_DELAYED_ITEMS_NR)
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04004826 nr = calc_reclaim_items_nr(fs_info, num_bytes) * 2;
Miao Xie18cd8ea2013-11-04 23:13:22 +08004827 else
Josef Bacik96c3f432012-06-21 14:05:49 -04004828 nr = -1;
Miao Xie18cd8ea2013-11-04 23:13:22 +08004829
Josef Bacik96c3f432012-06-21 14:05:49 -04004830 trans = btrfs_join_transaction(root);
4831 if (IS_ERR(trans)) {
4832 ret = PTR_ERR(trans);
4833 break;
4834 }
Nikolay Borisove5c304e62018-02-07 17:55:43 +02004835 ret = btrfs_run_delayed_items_nr(trans, nr);
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04004836 btrfs_end_transaction(trans);
Josef Bacik96c3f432012-06-21 14:05:49 -04004837 break;
Josef Bacik67b0fd62012-09-24 13:42:00 -04004838 case FLUSH_DELALLOC:
4839 case FLUSH_DELALLOC_WAIT:
Nikolay Borisov7bdd6272017-07-11 13:25:13 +03004840 shrink_delalloc(fs_info, num_bytes * 2, num_bytes,
Josef Bacik67b0fd62012-09-24 13:42:00 -04004841 state == FLUSH_DELALLOC_WAIT);
4842 break;
Josef Bacik413df722018-12-03 10:20:35 -05004843 case FLUSH_DELAYED_REFS_NR:
4844 case FLUSH_DELAYED_REFS:
4845 trans = btrfs_join_transaction(root);
4846 if (IS_ERR(trans)) {
4847 ret = PTR_ERR(trans);
4848 break;
4849 }
4850 if (state == FLUSH_DELAYED_REFS_NR)
4851 nr = calc_reclaim_items_nr(fs_info, num_bytes);
4852 else
4853 nr = 0;
4854 btrfs_run_delayed_refs(trans, nr);
4855 btrfs_end_transaction(trans);
4856 break;
Josef Bacikea658ba2012-09-11 16:57:25 -04004857 case ALLOC_CHUNK:
Josef Bacik450114f2018-11-21 14:03:08 -05004858 case ALLOC_CHUNK_FORCE:
Josef Bacikea658ba2012-09-11 16:57:25 -04004859 trans = btrfs_join_transaction(root);
4860 if (IS_ERR(trans)) {
4861 ret = PTR_ERR(trans);
4862 break;
4863 }
Nikolay Borisov01458822018-06-20 15:49:05 +03004864 ret = do_chunk_alloc(trans,
Jeff Mahoney1b868262017-05-17 11:38:35 -04004865 btrfs_metadata_alloc_profile(fs_info),
Josef Bacik450114f2018-11-21 14:03:08 -05004866 (state == ALLOC_CHUNK) ?
4867 CHUNK_ALLOC_NO_FORCE : CHUNK_ALLOC_FORCE);
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04004868 btrfs_end_transaction(trans);
Alex Lyakaseecba892015-12-06 12:32:31 +02004869 if (ret > 0 || ret == -ENOSPC)
Josef Bacikea658ba2012-09-11 16:57:25 -04004870 ret = 0;
4871 break;
Josef Bacik96c3f432012-06-21 14:05:49 -04004872 case COMMIT_TRANS:
Josef Bacik3ec9a4c2019-01-11 10:21:01 -05004873 /*
4874 * If we have pending delayed iputs then we could free up a
4875 * bunch of pinned space, so make sure we run the iputs before
4876 * we do our pinned bytes check below.
4877 */
Josef Bacik3ec9a4c2019-01-11 10:21:01 -05004878 btrfs_run_delayed_iputs(fs_info);
Josef Bacik034f7842018-12-03 11:06:52 -05004879 btrfs_wait_on_delayed_iputs(fs_info);
Josef Bacik3ec9a4c2019-01-11 10:21:01 -05004880
Josef Bacik996478c2017-08-22 16:00:39 -04004881 ret = may_commit_transaction(fs_info, space_info);
Josef Bacik96c3f432012-06-21 14:05:49 -04004882 break;
4883 default:
4884 ret = -ENOSPC;
4885 break;
4886 }
4887
Nikolay Borisov7bdd6272017-07-11 13:25:13 +03004888 trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, state,
4889 ret);
Nikolay Borisove38ae7a2017-07-25 17:48:28 +03004890 return;
Josef Bacik96c3f432012-06-21 14:05:49 -04004891}
Miao Xie21c7e752014-05-13 17:29:04 -07004892
4893static inline u64
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004894btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info,
4895 struct btrfs_space_info *space_info,
4896 bool system_chunk)
Miao Xie21c7e752014-05-13 17:29:04 -07004897{
Josef Bacik957780e2016-05-17 13:30:55 -04004898 struct reserve_ticket *ticket;
Miao Xie21c7e752014-05-13 17:29:04 -07004899 u64 used;
4900 u64 expected;
Josef Bacik957780e2016-05-17 13:30:55 -04004901 u64 to_reclaim = 0;
Miao Xie21c7e752014-05-13 17:29:04 -07004902
Josef Bacik957780e2016-05-17 13:30:55 -04004903 list_for_each_entry(ticket, &space_info->tickets, list)
4904 to_reclaim += ticket->bytes;
4905 list_for_each_entry(ticket, &space_info->priority_tickets, list)
4906 to_reclaim += ticket->bytes;
4907 if (to_reclaim)
4908 return to_reclaim;
Miao Xie21c7e752014-05-13 17:29:04 -07004909
Wang Xiaoguange0af2482016-08-31 19:46:16 +08004910 to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M);
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004911 if (can_overcommit(fs_info, space_info, to_reclaim,
4912 BTRFS_RESERVE_FLUSH_ALL, system_chunk))
Wang Xiaoguange0af2482016-08-31 19:46:16 +08004913 return 0;
4914
Nikolay Borisov0eee8a42017-06-14 11:35:34 +03004915 used = btrfs_space_info_used(space_info, true);
4916
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004917 if (can_overcommit(fs_info, space_info, SZ_1M,
4918 BTRFS_RESERVE_FLUSH_ALL, system_chunk))
Miao Xie21c7e752014-05-13 17:29:04 -07004919 expected = div_factor_fine(space_info->total_bytes, 95);
4920 else
4921 expected = div_factor_fine(space_info->total_bytes, 90);
4922
4923 if (used > expected)
4924 to_reclaim = used - expected;
4925 else
4926 to_reclaim = 0;
4927 to_reclaim = min(to_reclaim, space_info->bytes_may_use +
4928 space_info->bytes_reserved);
Miao Xie21c7e752014-05-13 17:29:04 -07004929 return to_reclaim;
4930}
4931
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004932static inline int need_do_async_reclaim(struct btrfs_fs_info *fs_info,
4933 struct btrfs_space_info *space_info,
4934 u64 used, bool system_chunk)
Miao Xie21c7e752014-05-13 17:29:04 -07004935{
Josef Bacik365c5312015-02-18 13:58:15 -08004936 u64 thresh = div_factor_fine(space_info->total_bytes, 98);
4937
4938 /* If we're just plain full then async reclaim just slows us down. */
Josef Bacikbaee8792016-01-26 09:35:38 -05004939 if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh)
Josef Bacik365c5312015-02-18 13:58:15 -08004940 return 0;
4941
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004942 if (!btrfs_calc_reclaim_metadata_size(fs_info, space_info,
4943 system_chunk))
Liu Bo25ce4592014-09-10 12:58:50 +08004944 return 0;
Liu Bo25ce4592014-09-10 12:58:50 +08004945
Jeff Mahoney0b246af2016-06-22 18:54:23 -04004946 return (used >= thresh && !btrfs_fs_closing(fs_info) &&
4947 !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state));
Miao Xie21c7e752014-05-13 17:29:04 -07004948}
4949
Josef Bacikf91587e2018-11-21 14:03:10 -05004950static bool wake_all_tickets(struct list_head *head)
Miao Xie21c7e752014-05-13 17:29:04 -07004951{
Josef Bacik957780e2016-05-17 13:30:55 -04004952 struct reserve_ticket *ticket;
Miao Xie21c7e752014-05-13 17:29:04 -07004953
Josef Bacik957780e2016-05-17 13:30:55 -04004954 while (!list_empty(head)) {
4955 ticket = list_first_entry(head, struct reserve_ticket, list);
4956 list_del_init(&ticket->list);
4957 ticket->error = -ENOSPC;
4958 wake_up(&ticket->wait);
Josef Bacikf91587e2018-11-21 14:03:10 -05004959 if (ticket->bytes != ticket->orig_bytes)
4960 return true;
Miao Xie21c7e752014-05-13 17:29:04 -07004961 }
Josef Bacikf91587e2018-11-21 14:03:10 -05004962 return false;
Miao Xie21c7e752014-05-13 17:29:04 -07004963}
4964
Josef Bacik957780e2016-05-17 13:30:55 -04004965/*
4966 * This is for normal flushers, we can wait all goddamned day if we want to. We
4967 * will loop and continuously try to flush as long as we are making progress.
4968 * We count progress as clearing off tickets each time we have to loop.
4969 */
Miao Xie21c7e752014-05-13 17:29:04 -07004970static void btrfs_async_reclaim_metadata_space(struct work_struct *work)
4971{
4972 struct btrfs_fs_info *fs_info;
4973 struct btrfs_space_info *space_info;
4974 u64 to_reclaim;
4975 int flush_state;
Josef Bacik957780e2016-05-17 13:30:55 -04004976 int commit_cycles = 0;
Wang Xiaoguangce129652016-09-02 10:58:46 +08004977 u64 last_tickets_id;
Miao Xie21c7e752014-05-13 17:29:04 -07004978
4979 fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work);
4980 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4981
Josef Bacik957780e2016-05-17 13:30:55 -04004982 spin_lock(&space_info->lock);
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04004983 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info,
4984 false);
Josef Bacik957780e2016-05-17 13:30:55 -04004985 if (!to_reclaim) {
4986 space_info->flush = 0;
4987 spin_unlock(&space_info->lock);
Miao Xie21c7e752014-05-13 17:29:04 -07004988 return;
Josef Bacik957780e2016-05-17 13:30:55 -04004989 }
Wang Xiaoguangce129652016-09-02 10:58:46 +08004990 last_tickets_id = space_info->tickets_id;
Josef Bacik957780e2016-05-17 13:30:55 -04004991 spin_unlock(&space_info->lock);
Miao Xie21c7e752014-05-13 17:29:04 -07004992
4993 flush_state = FLUSH_DELAYED_ITEMS_NR;
4994 do {
Nikolay Borisove38ae7a2017-07-25 17:48:28 +03004995 flush_space(fs_info, space_info, to_reclaim, flush_state);
Josef Bacik957780e2016-05-17 13:30:55 -04004996 spin_lock(&space_info->lock);
4997 if (list_empty(&space_info->tickets)) {
4998 space_info->flush = 0;
4999 spin_unlock(&space_info->lock);
Miao Xie21c7e752014-05-13 17:29:04 -07005000 return;
Josef Bacik957780e2016-05-17 13:30:55 -04005001 }
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005002 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info,
5003 space_info,
5004 false);
Wang Xiaoguangce129652016-09-02 10:58:46 +08005005 if (last_tickets_id == space_info->tickets_id) {
Josef Bacik957780e2016-05-17 13:30:55 -04005006 flush_state++;
5007 } else {
Wang Xiaoguangce129652016-09-02 10:58:46 +08005008 last_tickets_id = space_info->tickets_id;
Josef Bacik957780e2016-05-17 13:30:55 -04005009 flush_state = FLUSH_DELAYED_ITEMS_NR;
5010 if (commit_cycles)
5011 commit_cycles--;
5012 }
5013
Josef Bacik450114f2018-11-21 14:03:08 -05005014 /*
5015 * We don't want to force a chunk allocation until we've tried
5016 * pretty hard to reclaim space. Think of the case where we
5017 * freed up a bunch of space and so have a lot of pinned space
5018 * to reclaim. We would rather use that than possibly create a
5019 * underutilized metadata chunk. So if this is our first run
5020 * through the flushing state machine skip ALLOC_CHUNK_FORCE and
5021 * commit the transaction. If nothing has changed the next go
5022 * around then we can force a chunk allocation.
5023 */
5024 if (flush_state == ALLOC_CHUNK_FORCE && !commit_cycles)
5025 flush_state++;
5026
Josef Bacik957780e2016-05-17 13:30:55 -04005027 if (flush_state > COMMIT_TRANS) {
5028 commit_cycles++;
5029 if (commit_cycles > 2) {
Josef Bacikf91587e2018-11-21 14:03:10 -05005030 if (wake_all_tickets(&space_info->tickets)) {
5031 flush_state = FLUSH_DELAYED_ITEMS_NR;
5032 commit_cycles--;
5033 } else {
5034 space_info->flush = 0;
5035 }
Josef Bacik957780e2016-05-17 13:30:55 -04005036 } else {
5037 flush_state = FLUSH_DELAYED_ITEMS_NR;
5038 }
5039 }
5040 spin_unlock(&space_info->lock);
5041 } while (flush_state <= COMMIT_TRANS);
Miao Xie21c7e752014-05-13 17:29:04 -07005042}
5043
5044void btrfs_init_async_reclaim_work(struct work_struct *work)
5045{
5046 INIT_WORK(work, btrfs_async_reclaim_metadata_space);
5047}
5048
Josef Bacik8a1bbe12018-11-21 14:03:12 -05005049static const enum btrfs_flush_state priority_flush_states[] = {
5050 FLUSH_DELAYED_ITEMS_NR,
5051 FLUSH_DELAYED_ITEMS,
5052 ALLOC_CHUNK,
5053};
5054
Josef Bacik957780e2016-05-17 13:30:55 -04005055static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info,
5056 struct btrfs_space_info *space_info,
5057 struct reserve_ticket *ticket)
5058{
5059 u64 to_reclaim;
Josef Bacik8a1bbe12018-11-21 14:03:12 -05005060 int flush_state;
Josef Bacik957780e2016-05-17 13:30:55 -04005061
5062 spin_lock(&space_info->lock);
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005063 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info,
5064 false);
Josef Bacik957780e2016-05-17 13:30:55 -04005065 if (!to_reclaim) {
5066 spin_unlock(&space_info->lock);
5067 return;
5068 }
5069 spin_unlock(&space_info->lock);
5070
Josef Bacik8a1bbe12018-11-21 14:03:12 -05005071 flush_state = 0;
Josef Bacik957780e2016-05-17 13:30:55 -04005072 do {
Josef Bacik8a1bbe12018-11-21 14:03:12 -05005073 flush_space(fs_info, space_info, to_reclaim,
5074 priority_flush_states[flush_state]);
Josef Bacik957780e2016-05-17 13:30:55 -04005075 flush_state++;
5076 spin_lock(&space_info->lock);
5077 if (ticket->bytes == 0) {
5078 spin_unlock(&space_info->lock);
5079 return;
5080 }
5081 spin_unlock(&space_info->lock);
Josef Bacik8a1bbe12018-11-21 14:03:12 -05005082 } while (flush_state < ARRAY_SIZE(priority_flush_states));
Josef Bacik957780e2016-05-17 13:30:55 -04005083}
5084
5085static int wait_reserve_ticket(struct btrfs_fs_info *fs_info,
5086 struct btrfs_space_info *space_info,
Josef Bacikf91587e2018-11-21 14:03:10 -05005087 struct reserve_ticket *ticket)
Josef Bacik957780e2016-05-17 13:30:55 -04005088
5089{
5090 DEFINE_WAIT(wait);
Josef Bacikf91587e2018-11-21 14:03:10 -05005091 u64 reclaim_bytes = 0;
Josef Bacik957780e2016-05-17 13:30:55 -04005092 int ret = 0;
5093
5094 spin_lock(&space_info->lock);
5095 while (ticket->bytes > 0 && ticket->error == 0) {
5096 ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE);
5097 if (ret) {
5098 ret = -EINTR;
5099 break;
5100 }
5101 spin_unlock(&space_info->lock);
5102
5103 schedule();
5104
5105 finish_wait(&ticket->wait, &wait);
5106 spin_lock(&space_info->lock);
5107 }
5108 if (!ret)
5109 ret = ticket->error;
5110 if (!list_empty(&ticket->list))
5111 list_del_init(&ticket->list);
Josef Bacikf91587e2018-11-21 14:03:10 -05005112 if (ticket->bytes && ticket->bytes < ticket->orig_bytes)
5113 reclaim_bytes = ticket->orig_bytes - ticket->bytes;
Josef Bacik957780e2016-05-17 13:30:55 -04005114 spin_unlock(&space_info->lock);
5115
Josef Bacikf91587e2018-11-21 14:03:10 -05005116 if (reclaim_bytes)
5117 space_info_add_old_bytes(fs_info, space_info, reclaim_bytes);
Josef Bacik957780e2016-05-17 13:30:55 -04005118 return ret;
5119}
5120
Josef Bacik663350a2011-11-03 22:54:25 -04005121/**
Josef Bacik4a92b1b2011-08-30 12:34:28 -04005122 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
5123 * @root - the root we're allocating for
Josef Bacik957780e2016-05-17 13:30:55 -04005124 * @space_info - the space info we want to allocate from
Josef Bacik4a92b1b2011-08-30 12:34:28 -04005125 * @orig_bytes - the number of bytes we want
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04005126 * @flush - whether or not we can flush to make our reservation
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005127 *
Nicholas D Steeves01327612016-05-19 21:18:45 -04005128 * This will reserve orig_bytes number of bytes from the space info associated
Josef Bacik4a92b1b2011-08-30 12:34:28 -04005129 * with the block_rsv. If there is not enough space it will make an attempt to
5130 * flush out space to make room. It will do this by flushing delalloc if
5131 * possible or committing the transaction. If flush is 0 then no attempts to
5132 * regain reservations will be made and this will fail if there is not enough
5133 * space already.
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005134 */
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005135static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info,
Josef Bacik957780e2016-05-17 13:30:55 -04005136 struct btrfs_space_info *space_info,
5137 u64 orig_bytes,
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005138 enum btrfs_reserve_flush_enum flush,
5139 bool system_chunk)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005140{
Josef Bacik957780e2016-05-17 13:30:55 -04005141 struct reserve_ticket ticket;
Josef Bacik2bf64752011-09-26 17:12:22 -04005142 u64 used;
Josef Bacikf91587e2018-11-21 14:03:10 -05005143 u64 reclaim_bytes = 0;
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005144 int ret = 0;
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005145
Josef Bacik957780e2016-05-17 13:30:55 -04005146 ASSERT(orig_bytes);
Josef Bacik8ca17f02016-05-27 13:24:13 -04005147 ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_ALL);
5148
Yan, Zhengf0486c62010-05-16 10:46:25 -04005149 spin_lock(&space_info->lock);
Josef Bacikfdb5eff2011-06-07 16:07:44 -04005150 ret = -ENOSPC;
Liu Bo41361352017-02-13 15:42:21 -08005151 used = btrfs_space_info_used(space_info, true);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005152
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005153 /*
Josef Bacik957780e2016-05-17 13:30:55 -04005154 * If we have enough space then hooray, make our reservation and carry
5155 * on. If not see if we can overcommit, and if we can, hooray carry on.
5156 * If not things get more complicated.
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005157 */
Josef Bacik957780e2016-05-17 13:30:55 -04005158 if (used + orig_bytes <= space_info->total_bytes) {
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08005159 update_bytes_may_use(space_info, orig_bytes);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005160 trace_btrfs_space_reservation(fs_info, "space_info",
5161 space_info->flags, orig_bytes, 1);
Josef Bacik957780e2016-05-17 13:30:55 -04005162 ret = 0;
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005163 } else if (can_overcommit(fs_info, space_info, orig_bytes, flush,
5164 system_chunk)) {
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08005165 update_bytes_may_use(space_info, orig_bytes);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005166 trace_btrfs_space_reservation(fs_info, "space_info",
5167 space_info->flags, orig_bytes, 1);
Josef Bacik44734ed2012-09-28 16:04:19 -04005168 ret = 0;
Josef Bacik2bf64752011-09-26 17:12:22 -04005169 }
5170
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005171 /*
Josef Bacik957780e2016-05-17 13:30:55 -04005172 * If we couldn't make a reservation then setup our reservation ticket
5173 * and kick the async worker if it's not already running.
Miao Xie08e007d2012-10-16 11:33:38 +00005174 *
Josef Bacik957780e2016-05-17 13:30:55 -04005175 * If we are a priority flusher then we just need to add our ticket to
5176 * the list and we will do our own flushing further down.
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005177 */
Josef Bacik72bcd992012-12-18 15:16:34 -05005178 if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
Josef Bacikf91587e2018-11-21 14:03:10 -05005179 ticket.orig_bytes = orig_bytes;
Josef Bacik957780e2016-05-17 13:30:55 -04005180 ticket.bytes = orig_bytes;
5181 ticket.error = 0;
5182 init_waitqueue_head(&ticket.wait);
5183 if (flush == BTRFS_RESERVE_FLUSH_ALL) {
5184 list_add_tail(&ticket.list, &space_info->tickets);
5185 if (!space_info->flush) {
5186 space_info->flush = 1;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005187 trace_btrfs_trigger_flush(fs_info,
Josef Bacikf376df22016-03-25 13:25:56 -04005188 space_info->flags,
5189 orig_bytes, flush,
5190 "enospc");
Josef Bacik957780e2016-05-17 13:30:55 -04005191 queue_work(system_unbound_wq,
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005192 &fs_info->async_reclaim_work);
Josef Bacik957780e2016-05-17 13:30:55 -04005193 }
5194 } else {
5195 list_add_tail(&ticket.list,
5196 &space_info->priority_tickets);
5197 }
Miao Xie21c7e752014-05-13 17:29:04 -07005198 } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {
5199 used += orig_bytes;
Josef Bacikf6acfd52014-09-18 11:27:17 -04005200 /*
5201 * We will do the space reservation dance during log replay,
5202 * which means we won't have fs_info->fs_root set, so don't do
5203 * the async reclaim as we will panic.
5204 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005205 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) &&
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005206 need_do_async_reclaim(fs_info, space_info,
5207 used, system_chunk) &&
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005208 !work_busy(&fs_info->async_reclaim_work)) {
5209 trace_btrfs_trigger_flush(fs_info, space_info->flags,
5210 orig_bytes, flush, "preempt");
Miao Xie21c7e752014-05-13 17:29:04 -07005211 queue_work(system_unbound_wq,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005212 &fs_info->async_reclaim_work);
Josef Bacikf376df22016-03-25 13:25:56 -04005213 }
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005214 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04005215 spin_unlock(&space_info->lock);
Miao Xie08e007d2012-10-16 11:33:38 +00005216 if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
Josef Bacik957780e2016-05-17 13:30:55 -04005217 return ret;
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005218
Josef Bacik957780e2016-05-17 13:30:55 -04005219 if (flush == BTRFS_RESERVE_FLUSH_ALL)
Josef Bacikf91587e2018-11-21 14:03:10 -05005220 return wait_reserve_ticket(fs_info, space_info, &ticket);
Miao Xie08e007d2012-10-16 11:33:38 +00005221
Josef Bacik957780e2016-05-17 13:30:55 -04005222 ret = 0;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005223 priority_reclaim_metadata_space(fs_info, space_info, &ticket);
Josef Bacik957780e2016-05-17 13:30:55 -04005224 spin_lock(&space_info->lock);
5225 if (ticket.bytes) {
Josef Bacikf91587e2018-11-21 14:03:10 -05005226 if (ticket.bytes < orig_bytes)
5227 reclaim_bytes = orig_bytes - ticket.bytes;
Josef Bacik957780e2016-05-17 13:30:55 -04005228 list_del_init(&ticket.list);
5229 ret = -ENOSPC;
5230 }
5231 spin_unlock(&space_info->lock);
Josef Bacikf91587e2018-11-21 14:03:10 -05005232
5233 if (reclaim_bytes)
5234 space_info_add_old_bytes(fs_info, space_info, reclaim_bytes);
Josef Bacik957780e2016-05-17 13:30:55 -04005235 ASSERT(list_empty(&ticket.list));
5236 return ret;
5237}
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005238
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005239/**
5240 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
5241 * @root - the root we're allocating for
5242 * @block_rsv - the block_rsv we're allocating for
5243 * @orig_bytes - the number of bytes we want
5244 * @flush - whether or not we can flush to make our reservation
5245 *
Andrea Gelmini52042d82018-11-28 12:05:13 +01005246 * This will reserve orig_bytes number of bytes from the space info associated
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005247 * with the block_rsv. If there is not enough space it will make an attempt to
5248 * flush out space to make room. It will do this by flushing delalloc if
5249 * possible or committing the transaction. If flush is 0 then no attempts to
5250 * regain reservations will be made and this will fail if there is not enough
5251 * space already.
5252 */
5253static int reserve_metadata_bytes(struct btrfs_root *root,
5254 struct btrfs_block_rsv *block_rsv,
5255 u64 orig_bytes,
5256 enum btrfs_reserve_flush_enum flush)
5257{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005258 struct btrfs_fs_info *fs_info = root->fs_info;
5259 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
Josef Bacik957780e2016-05-17 13:30:55 -04005260 int ret;
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005261 bool system_chunk = (root == fs_info->chunk_root);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005262
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005263 ret = __reserve_metadata_bytes(fs_info, block_rsv->space_info,
5264 orig_bytes, flush, system_chunk);
Josef Bacik5d803662013-02-07 16:06:02 -05005265 if (ret == -ENOSPC &&
5266 unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) {
Josef Bacik5d803662013-02-07 16:06:02 -05005267 if (block_rsv != global_rsv &&
5268 !block_rsv_use_bytes(global_rsv, orig_bytes))
5269 ret = 0;
5270 }
Nikolay Borisov9a3daff2017-12-15 12:05:37 +02005271 if (ret == -ENOSPC) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005272 trace_btrfs_space_reservation(fs_info, "space_info:enospc",
Josef Bacik957780e2016-05-17 13:30:55 -04005273 block_rsv->space_info->flags,
5274 orig_bytes, 1);
Nikolay Borisov9a3daff2017-12-15 12:05:37 +02005275
5276 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG))
5277 dump_space_info(fs_info, block_rsv->space_info,
5278 orig_bytes, 0);
5279 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04005280 return ret;
5281}
5282
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005283static struct btrfs_block_rsv *get_block_rsv(
5284 const struct btrfs_trans_handle *trans,
5285 const struct btrfs_root *root)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005286{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005287 struct btrfs_fs_info *fs_info = root->fs_info;
Josef Bacik4c13d752011-08-30 11:31:29 -04005288 struct btrfs_block_rsv *block_rsv = NULL;
5289
Alexandru Moisee9cf4392015-09-09 00:18:50 +00005290 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005291 (root == fs_info->csum_root && trans->adding_csums) ||
5292 (root == fs_info->uuid_root))
Stefan Behrensf7a81ea2013-08-15 17:11:19 +02005293 block_rsv = trans->block_rsv;
5294
Josef Bacik4c13d752011-08-30 11:31:29 -04005295 if (!block_rsv)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005296 block_rsv = root->block_rsv;
5297
5298 if (!block_rsv)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005299 block_rsv = &fs_info->empty_block_rsv;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005300
5301 return block_rsv;
5302}
5303
5304static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
5305 u64 num_bytes)
5306{
5307 int ret = -ENOSPC;
5308 spin_lock(&block_rsv->lock);
5309 if (block_rsv->reserved >= num_bytes) {
5310 block_rsv->reserved -= num_bytes;
5311 if (block_rsv->reserved < block_rsv->size)
5312 block_rsv->full = 0;
5313 ret = 0;
5314 }
5315 spin_unlock(&block_rsv->lock);
5316 return ret;
5317}
5318
5319static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
Lu Fengqi3a584172018-08-04 21:10:55 +08005320 u64 num_bytes, bool update_size)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005321{
5322 spin_lock(&block_rsv->lock);
5323 block_rsv->reserved += num_bytes;
5324 if (update_size)
5325 block_rsv->size += num_bytes;
5326 else if (block_rsv->reserved >= block_rsv->size)
5327 block_rsv->full = 1;
5328 spin_unlock(&block_rsv->lock);
5329}
5330
Josef Bacikd52be812013-05-29 14:54:47 -04005331int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info,
5332 struct btrfs_block_rsv *dest, u64 num_bytes,
5333 int min_factor)
5334{
5335 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5336 u64 min_bytes;
5337
5338 if (global_rsv->space_info != dest->space_info)
5339 return -ENOSPC;
5340
5341 spin_lock(&global_rsv->lock);
5342 min_bytes = div_factor(global_rsv->size, min_factor);
5343 if (global_rsv->reserved < min_bytes + num_bytes) {
5344 spin_unlock(&global_rsv->lock);
5345 return -ENOSPC;
5346 }
5347 global_rsv->reserved -= num_bytes;
5348 if (global_rsv->reserved < global_rsv->size)
5349 global_rsv->full = 0;
5350 spin_unlock(&global_rsv->lock);
5351
Lu Fengqi3a584172018-08-04 21:10:55 +08005352 block_rsv_add_bytes(dest, num_bytes, true);
Josef Bacikd52be812013-05-29 14:54:47 -04005353 return 0;
5354}
5355
Josef Bacikba2c4d42018-12-03 10:20:33 -05005356/**
5357 * btrfs_migrate_to_delayed_refs_rsv - transfer bytes to our delayed refs rsv.
5358 * @fs_info - the fs info for our fs.
5359 * @src - the source block rsv to transfer from.
5360 * @num_bytes - the number of bytes to transfer.
5361 *
5362 * This transfers up to the num_bytes amount from the src rsv to the
5363 * delayed_refs_rsv. Any extra bytes are returned to the space info.
5364 */
5365void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info,
5366 struct btrfs_block_rsv *src,
5367 u64 num_bytes)
5368{
5369 struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv;
5370 u64 to_free = 0;
5371
5372 spin_lock(&src->lock);
5373 src->reserved -= num_bytes;
5374 src->size -= num_bytes;
5375 spin_unlock(&src->lock);
5376
5377 spin_lock(&delayed_refs_rsv->lock);
5378 if (delayed_refs_rsv->size > delayed_refs_rsv->reserved) {
5379 u64 delta = delayed_refs_rsv->size -
5380 delayed_refs_rsv->reserved;
5381 if (num_bytes > delta) {
5382 to_free = num_bytes - delta;
5383 num_bytes = delta;
5384 }
5385 } else {
5386 to_free = num_bytes;
5387 num_bytes = 0;
5388 }
5389
5390 if (num_bytes)
5391 delayed_refs_rsv->reserved += num_bytes;
5392 if (delayed_refs_rsv->reserved >= delayed_refs_rsv->size)
5393 delayed_refs_rsv->full = 1;
5394 spin_unlock(&delayed_refs_rsv->lock);
5395
5396 if (num_bytes)
5397 trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
5398 0, num_bytes, 1);
5399 if (to_free)
5400 space_info_add_old_bytes(fs_info, delayed_refs_rsv->space_info,
5401 to_free);
5402}
5403
5404/**
5405 * btrfs_delayed_refs_rsv_refill - refill based on our delayed refs usage.
5406 * @fs_info - the fs_info for our fs.
5407 * @flush - control how we can flush for this reservation.
5408 *
5409 * This will refill the delayed block_rsv up to 1 items size worth of space and
5410 * will return -ENOSPC if we can't make the reservation.
5411 */
5412int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info,
5413 enum btrfs_reserve_flush_enum flush)
5414{
5415 struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv;
5416 u64 limit = btrfs_calc_trans_metadata_size(fs_info, 1);
5417 u64 num_bytes = 0;
5418 int ret = -ENOSPC;
5419
5420 spin_lock(&block_rsv->lock);
5421 if (block_rsv->reserved < block_rsv->size) {
5422 num_bytes = block_rsv->size - block_rsv->reserved;
5423 num_bytes = min(num_bytes, limit);
5424 }
5425 spin_unlock(&block_rsv->lock);
5426
5427 if (!num_bytes)
5428 return 0;
5429
5430 ret = reserve_metadata_bytes(fs_info->extent_root, block_rsv,
5431 num_bytes, flush);
5432 if (ret)
5433 return ret;
5434 block_rsv_add_bytes(block_rsv, num_bytes, 0);
5435 trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
5436 0, num_bytes, 1);
5437 return 0;
5438}
5439
Josef Bacik957780e2016-05-17 13:30:55 -04005440/*
5441 * This is for space we already have accounted in space_info->bytes_may_use, so
5442 * basically when we're returning space from block_rsv's.
5443 */
5444static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info,
5445 struct btrfs_space_info *space_info,
5446 u64 num_bytes)
5447{
5448 struct reserve_ticket *ticket;
5449 struct list_head *head;
5450 u64 used;
5451 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH;
5452 bool check_overcommit = false;
5453
5454 spin_lock(&space_info->lock);
5455 head = &space_info->priority_tickets;
5456
5457 /*
5458 * If we are over our limit then we need to check and see if we can
5459 * overcommit, and if we can't then we just need to free up our space
5460 * and not satisfy any requests.
5461 */
Nikolay Borisov0eee8a42017-06-14 11:35:34 +03005462 used = btrfs_space_info_used(space_info, true);
Josef Bacik957780e2016-05-17 13:30:55 -04005463 if (used - num_bytes >= space_info->total_bytes)
5464 check_overcommit = true;
5465again:
5466 while (!list_empty(head) && num_bytes) {
5467 ticket = list_first_entry(head, struct reserve_ticket,
5468 list);
5469 /*
5470 * We use 0 bytes because this space is already reserved, so
5471 * adding the ticket space would be a double count.
5472 */
5473 if (check_overcommit &&
Jeff Mahoneyc1c49192017-05-17 11:38:36 -04005474 !can_overcommit(fs_info, space_info, 0, flush, false))
Josef Bacik957780e2016-05-17 13:30:55 -04005475 break;
5476 if (num_bytes >= ticket->bytes) {
5477 list_del_init(&ticket->list);
5478 num_bytes -= ticket->bytes;
5479 ticket->bytes = 0;
Wang Xiaoguangce129652016-09-02 10:58:46 +08005480 space_info->tickets_id++;
Josef Bacik957780e2016-05-17 13:30:55 -04005481 wake_up(&ticket->wait);
5482 } else {
5483 ticket->bytes -= num_bytes;
5484 num_bytes = 0;
5485 }
5486 }
5487
5488 if (num_bytes && head == &space_info->priority_tickets) {
5489 head = &space_info->tickets;
5490 flush = BTRFS_RESERVE_FLUSH_ALL;
5491 goto again;
5492 }
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08005493 update_bytes_may_use(space_info, -num_bytes);
Josef Bacik957780e2016-05-17 13:30:55 -04005494 trace_btrfs_space_reservation(fs_info, "space_info",
5495 space_info->flags, num_bytes, 0);
5496 spin_unlock(&space_info->lock);
5497}
5498
5499/*
5500 * This is for newly allocated space that isn't accounted in
5501 * space_info->bytes_may_use yet. So if we allocate a chunk or unpin an extent
5502 * we use this helper.
5503 */
5504static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info,
5505 struct btrfs_space_info *space_info,
5506 u64 num_bytes)
5507{
5508 struct reserve_ticket *ticket;
5509 struct list_head *head = &space_info->priority_tickets;
5510
5511again:
5512 while (!list_empty(head) && num_bytes) {
5513 ticket = list_first_entry(head, struct reserve_ticket,
5514 list);
5515 if (num_bytes >= ticket->bytes) {
5516 trace_btrfs_space_reservation(fs_info, "space_info",
5517 space_info->flags,
5518 ticket->bytes, 1);
5519 list_del_init(&ticket->list);
5520 num_bytes -= ticket->bytes;
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08005521 update_bytes_may_use(space_info, ticket->bytes);
Josef Bacik957780e2016-05-17 13:30:55 -04005522 ticket->bytes = 0;
Wang Xiaoguangce129652016-09-02 10:58:46 +08005523 space_info->tickets_id++;
Josef Bacik957780e2016-05-17 13:30:55 -04005524 wake_up(&ticket->wait);
5525 } else {
5526 trace_btrfs_space_reservation(fs_info, "space_info",
5527 space_info->flags,
5528 num_bytes, 1);
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08005529 update_bytes_may_use(space_info, num_bytes);
Josef Bacik957780e2016-05-17 13:30:55 -04005530 ticket->bytes -= num_bytes;
5531 num_bytes = 0;
5532 }
5533 }
5534
5535 if (num_bytes && head == &space_info->priority_tickets) {
5536 head = &space_info->tickets;
5537 goto again;
5538 }
5539}
5540
Josef Bacik69fe2d72017-10-19 14:15:57 -04005541static u64 block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
Josef Bacik8c2a3ca2012-01-10 10:31:31 -05005542 struct btrfs_block_rsv *block_rsv,
Qu Wenruoff6bc372017-12-21 13:42:04 +08005543 struct btrfs_block_rsv *dest, u64 num_bytes,
5544 u64 *qgroup_to_release_ret)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005545{
5546 struct btrfs_space_info *space_info = block_rsv->space_info;
Qu Wenruoff6bc372017-12-21 13:42:04 +08005547 u64 qgroup_to_release = 0;
Josef Bacik69fe2d72017-10-19 14:15:57 -04005548 u64 ret;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005549
5550 spin_lock(&block_rsv->lock);
Qu Wenruoff6bc372017-12-21 13:42:04 +08005551 if (num_bytes == (u64)-1) {
Yan, Zhengf0486c62010-05-16 10:46:25 -04005552 num_bytes = block_rsv->size;
Qu Wenruoff6bc372017-12-21 13:42:04 +08005553 qgroup_to_release = block_rsv->qgroup_rsv_size;
5554 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04005555 block_rsv->size -= num_bytes;
5556 if (block_rsv->reserved >= block_rsv->size) {
5557 num_bytes = block_rsv->reserved - block_rsv->size;
5558 block_rsv->reserved = block_rsv->size;
5559 block_rsv->full = 1;
5560 } else {
5561 num_bytes = 0;
5562 }
Qu Wenruoff6bc372017-12-21 13:42:04 +08005563 if (block_rsv->qgroup_rsv_reserved >= block_rsv->qgroup_rsv_size) {
5564 qgroup_to_release = block_rsv->qgroup_rsv_reserved -
5565 block_rsv->qgroup_rsv_size;
5566 block_rsv->qgroup_rsv_reserved = block_rsv->qgroup_rsv_size;
5567 } else {
5568 qgroup_to_release = 0;
5569 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04005570 spin_unlock(&block_rsv->lock);
5571
Josef Bacik69fe2d72017-10-19 14:15:57 -04005572 ret = num_bytes;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005573 if (num_bytes > 0) {
5574 if (dest) {
Josef Bacike9e22892011-01-24 21:43:19 +00005575 spin_lock(&dest->lock);
5576 if (!dest->full) {
5577 u64 bytes_to_add;
5578
5579 bytes_to_add = dest->size - dest->reserved;
5580 bytes_to_add = min(num_bytes, bytes_to_add);
5581 dest->reserved += bytes_to_add;
5582 if (dest->reserved >= dest->size)
5583 dest->full = 1;
5584 num_bytes -= bytes_to_add;
5585 }
5586 spin_unlock(&dest->lock);
5587 }
Josef Bacik957780e2016-05-17 13:30:55 -04005588 if (num_bytes)
5589 space_info_add_old_bytes(fs_info, space_info,
5590 num_bytes);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005591 }
Qu Wenruoff6bc372017-12-21 13:42:04 +08005592 if (qgroup_to_release_ret)
5593 *qgroup_to_release_ret = qgroup_to_release;
Josef Bacik69fe2d72017-10-19 14:15:57 -04005594 return ret;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005595}
5596
Josef Bacik25d609f2016-03-25 13:25:48 -04005597int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src,
5598 struct btrfs_block_rsv *dst, u64 num_bytes,
Lu Fengqi3a584172018-08-04 21:10:55 +08005599 bool update_size)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005600{
5601 int ret;
5602
5603 ret = block_rsv_use_bytes(src, num_bytes);
5604 if (ret)
5605 return ret;
5606
Josef Bacik25d609f2016-03-25 13:25:48 -04005607 block_rsv_add_bytes(dst, num_bytes, update_size);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005608 return 0;
5609}
5610
Miao Xie66d8f3d2012-09-06 04:02:28 -06005611void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005612{
5613 memset(rsv, 0, sizeof(*rsv));
5614 spin_lock_init(&rsv->lock);
Miao Xie66d8f3d2012-09-06 04:02:28 -06005615 rsv->type = type;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005616}
5617
Josef Bacik69fe2d72017-10-19 14:15:57 -04005618void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info,
5619 struct btrfs_block_rsv *rsv,
5620 unsigned short type)
5621{
5622 btrfs_init_block_rsv(rsv, type);
5623 rsv->space_info = __find_space_info(fs_info,
5624 BTRFS_BLOCK_GROUP_METADATA);
5625}
5626
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04005627struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info,
Miao Xie66d8f3d2012-09-06 04:02:28 -06005628 unsigned short type)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005629{
5630 struct btrfs_block_rsv *block_rsv;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005631
5632 block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
5633 if (!block_rsv)
5634 return NULL;
5635
Josef Bacik69fe2d72017-10-19 14:15:57 -04005636 btrfs_init_metadata_block_rsv(fs_info, block_rsv, type);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005637 return block_rsv;
5638}
5639
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04005640void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info,
Yan, Zhengf0486c62010-05-16 10:46:25 -04005641 struct btrfs_block_rsv *rsv)
5642{
Josef Bacik2aaa6652012-08-29 14:27:18 -04005643 if (!rsv)
5644 return;
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04005645 btrfs_block_rsv_release(fs_info, rsv, (u64)-1);
Josef Bacikdabdb642011-08-08 12:50:18 -04005646 kfree(rsv);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005647}
5648
Miao Xie08e007d2012-10-16 11:33:38 +00005649int btrfs_block_rsv_add(struct btrfs_root *root,
5650 struct btrfs_block_rsv *block_rsv, u64 num_bytes,
5651 enum btrfs_reserve_flush_enum flush)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005652{
5653 int ret;
5654
5655 if (num_bytes == 0)
5656 return 0;
Josef Bacik8bb8ab22010-10-15 16:52:49 -04005657
Miao Xie61b520a2011-11-10 20:45:05 -05005658 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
Lu Fengqi5a2cb252018-08-04 21:10:56 +08005659 if (!ret)
Lu Fengqi3a584172018-08-04 21:10:55 +08005660 block_rsv_add_bytes(block_rsv, num_bytes, true);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005661
Yan, Zhengf0486c62010-05-16 10:46:25 -04005662 return ret;
5663}
5664
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04005665int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005666{
5667 u64 num_bytes = 0;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005668 int ret = -ENOSPC;
5669
5670 if (!block_rsv)
5671 return 0;
5672
5673 spin_lock(&block_rsv->lock);
Josef Bacik36ba0222011-10-18 12:15:48 -04005674 num_bytes = div_factor(block_rsv->size, min_factor);
5675 if (block_rsv->reserved >= num_bytes)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005676 ret = 0;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005677 spin_unlock(&block_rsv->lock);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005678
Josef Bacik36ba0222011-10-18 12:15:48 -04005679 return ret;
5680}
5681
Miao Xie08e007d2012-10-16 11:33:38 +00005682int btrfs_block_rsv_refill(struct btrfs_root *root,
5683 struct btrfs_block_rsv *block_rsv, u64 min_reserved,
5684 enum btrfs_reserve_flush_enum flush)
Josef Bacik36ba0222011-10-18 12:15:48 -04005685{
5686 u64 num_bytes = 0;
5687 int ret = -ENOSPC;
5688
5689 if (!block_rsv)
5690 return 0;
5691
5692 spin_lock(&block_rsv->lock);
5693 num_bytes = min_reserved;
Josef Bacik13553e52011-08-08 13:33:21 -04005694 if (block_rsv->reserved >= num_bytes)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005695 ret = 0;
Josef Bacik13553e52011-08-08 13:33:21 -04005696 else
Yan, Zhengf0486c62010-05-16 10:46:25 -04005697 num_bytes -= block_rsv->reserved;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005698 spin_unlock(&block_rsv->lock);
Josef Bacik13553e52011-08-08 13:33:21 -04005699
Yan, Zhengf0486c62010-05-16 10:46:25 -04005700 if (!ret)
5701 return 0;
5702
Miao Xieaa38a712011-11-18 17:43:00 +08005703 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
Josef Bacikdabdb642011-08-08 12:50:18 -04005704 if (!ret) {
Lu Fengqi3a584172018-08-04 21:10:55 +08005705 block_rsv_add_bytes(block_rsv, num_bytes, false);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005706 return 0;
5707 }
5708
Josef Bacik13553e52011-08-08 13:33:21 -04005709 return ret;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005710}
5711
Josef Bacik5df11362018-11-21 14:03:11 -05005712static void calc_refill_bytes(struct btrfs_block_rsv *block_rsv,
5713 u64 *metadata_bytes, u64 *qgroup_bytes)
5714{
5715 *metadata_bytes = 0;
5716 *qgroup_bytes = 0;
5717
5718 spin_lock(&block_rsv->lock);
5719 if (block_rsv->reserved < block_rsv->size)
5720 *metadata_bytes = block_rsv->size - block_rsv->reserved;
5721 if (block_rsv->qgroup_rsv_reserved < block_rsv->qgroup_rsv_size)
5722 *qgroup_bytes = block_rsv->qgroup_rsv_size -
5723 block_rsv->qgroup_rsv_reserved;
5724 spin_unlock(&block_rsv->lock);
5725}
5726
Josef Bacik69fe2d72017-10-19 14:15:57 -04005727/**
5728 * btrfs_inode_rsv_refill - refill the inode block rsv.
5729 * @inode - the inode we are refilling.
Andrea Gelmini52042d82018-11-28 12:05:13 +01005730 * @flush - the flushing restriction.
Josef Bacik69fe2d72017-10-19 14:15:57 -04005731 *
5732 * Essentially the same as btrfs_block_rsv_refill, except it uses the
5733 * block_rsv->size as the minimum size. We'll either refill the missing amount
Andrea Gelmini52042d82018-11-28 12:05:13 +01005734 * or return if we already have enough space. This will also handle the reserve
Josef Bacik69fe2d72017-10-19 14:15:57 -04005735 * tracepoint for the reserved amount.
5736 */
Qu Wenruo3f2dd7a2017-11-17 15:14:19 +08005737static int btrfs_inode_rsv_refill(struct btrfs_inode *inode,
5738 enum btrfs_reserve_flush_enum flush)
Josef Bacik69fe2d72017-10-19 14:15:57 -04005739{
5740 struct btrfs_root *root = inode->root;
5741 struct btrfs_block_rsv *block_rsv = &inode->block_rsv;
Josef Bacik5df11362018-11-21 14:03:11 -05005742 u64 num_bytes, last = 0;
5743 u64 qgroup_num_bytes;
Josef Bacik69fe2d72017-10-19 14:15:57 -04005744 int ret = -ENOSPC;
5745
Josef Bacik5df11362018-11-21 14:03:11 -05005746 calc_refill_bytes(block_rsv, &num_bytes, &qgroup_num_bytes);
Josef Bacik69fe2d72017-10-19 14:15:57 -04005747 if (num_bytes == 0)
5748 return 0;
5749
Josef Bacik5df11362018-11-21 14:03:11 -05005750 do {
5751 ret = btrfs_qgroup_reserve_meta_prealloc(root, qgroup_num_bytes,
5752 true);
5753 if (ret)
5754 return ret;
5755 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
5756 if (ret) {
5757 btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes);
5758 last = num_bytes;
5759 /*
5760 * If we are fragmented we can end up with a lot of
5761 * outstanding extents which will make our size be much
5762 * larger than our reserved amount.
5763 *
5764 * If the reservation happens here, it might be very
5765 * big though not needed in the end, if the delalloc
5766 * flushing happens.
5767 *
5768 * If this is the case try and do the reserve again.
5769 */
5770 if (flush == BTRFS_RESERVE_FLUSH_ALL)
5771 calc_refill_bytes(block_rsv, &num_bytes,
5772 &qgroup_num_bytes);
5773 if (num_bytes == 0)
5774 return 0;
5775 }
5776 } while (ret && last != num_bytes);
5777
Josef Bacik69fe2d72017-10-19 14:15:57 -04005778 if (!ret) {
Lu Fengqi3a584172018-08-04 21:10:55 +08005779 block_rsv_add_bytes(block_rsv, num_bytes, false);
Josef Bacik69fe2d72017-10-19 14:15:57 -04005780 trace_btrfs_space_reservation(root->fs_info, "delalloc",
5781 btrfs_ino(inode), num_bytes, 1);
Qu Wenruoff6bc372017-12-21 13:42:04 +08005782
5783 /* Don't forget to increase qgroup_rsv_reserved */
5784 spin_lock(&block_rsv->lock);
5785 block_rsv->qgroup_rsv_reserved += qgroup_num_bytes;
5786 spin_unlock(&block_rsv->lock);
Josef Bacik5df11362018-11-21 14:03:11 -05005787 }
Josef Bacik69fe2d72017-10-19 14:15:57 -04005788 return ret;
5789}
5790
Josef Bacikba2c4d42018-12-03 10:20:33 -05005791static u64 __btrfs_block_rsv_release(struct btrfs_fs_info *fs_info,
5792 struct btrfs_block_rsv *block_rsv,
5793 u64 num_bytes, u64 *qgroup_to_release)
5794{
5795 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5796 struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv;
5797 struct btrfs_block_rsv *target = delayed_rsv;
5798
5799 if (target->full || target == block_rsv)
5800 target = global_rsv;
5801
5802 if (block_rsv->space_info != target->space_info)
5803 target = NULL;
5804
5805 return block_rsv_release_bytes(fs_info, block_rsv, target, num_bytes,
5806 qgroup_to_release);
5807}
5808
5809void btrfs_block_rsv_release(struct btrfs_fs_info *fs_info,
5810 struct btrfs_block_rsv *block_rsv,
5811 u64 num_bytes)
5812{
5813 __btrfs_block_rsv_release(fs_info, block_rsv, num_bytes, NULL);
5814}
5815
Josef Bacik69fe2d72017-10-19 14:15:57 -04005816/**
5817 * btrfs_inode_rsv_release - release any excessive reservation.
5818 * @inode - the inode we need to release from.
Qu Wenruo43b18592017-12-12 15:34:32 +08005819 * @qgroup_free - free or convert qgroup meta.
5820 * Unlike normal operation, qgroup meta reservation needs to know if we are
5821 * freeing qgroup reservation or just converting it into per-trans. Normally
5822 * @qgroup_free is true for error handling, and false for normal release.
Josef Bacik69fe2d72017-10-19 14:15:57 -04005823 *
5824 * This is the same as btrfs_block_rsv_release, except that it handles the
5825 * tracepoint for the reservation.
5826 */
Qu Wenruo43b18592017-12-12 15:34:32 +08005827static void btrfs_inode_rsv_release(struct btrfs_inode *inode, bool qgroup_free)
Josef Bacik69fe2d72017-10-19 14:15:57 -04005828{
5829 struct btrfs_fs_info *fs_info = inode->root->fs_info;
Josef Bacik69fe2d72017-10-19 14:15:57 -04005830 struct btrfs_block_rsv *block_rsv = &inode->block_rsv;
5831 u64 released = 0;
Qu Wenruoff6bc372017-12-21 13:42:04 +08005832 u64 qgroup_to_release = 0;
Josef Bacik69fe2d72017-10-19 14:15:57 -04005833
5834 /*
5835 * Since we statically set the block_rsv->size we just want to say we
5836 * are releasing 0 bytes, and then we'll just get the reservation over
5837 * the size free'd.
5838 */
Josef Bacikba2c4d42018-12-03 10:20:33 -05005839 released = __btrfs_block_rsv_release(fs_info, block_rsv, 0,
5840 &qgroup_to_release);
Josef Bacik69fe2d72017-10-19 14:15:57 -04005841 if (released > 0)
5842 trace_btrfs_space_reservation(fs_info, "delalloc",
5843 btrfs_ino(inode), released, 0);
Qu Wenruo43b18592017-12-12 15:34:32 +08005844 if (qgroup_free)
Qu Wenruoff6bc372017-12-21 13:42:04 +08005845 btrfs_qgroup_free_meta_prealloc(inode->root, qgroup_to_release);
Qu Wenruo43b18592017-12-12 15:34:32 +08005846 else
Qu Wenruoff6bc372017-12-21 13:42:04 +08005847 btrfs_qgroup_convert_reserved_meta(inode->root,
5848 qgroup_to_release);
Josef Bacik69fe2d72017-10-19 14:15:57 -04005849}
5850
Josef Bacikba2c4d42018-12-03 10:20:33 -05005851/**
5852 * btrfs_delayed_refs_rsv_release - release a ref head's reservation.
5853 * @fs_info - the fs_info for our fs.
5854 * @nr - the number of items to drop.
5855 *
5856 * This drops the delayed ref head's count from the delayed refs rsv and frees
5857 * any excess reservation we had.
5858 */
5859void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr)
Yan, Zhengf0486c62010-05-16 10:46:25 -04005860{
Josef Bacikba2c4d42018-12-03 10:20:33 -05005861 struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005862 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
Josef Bacikba2c4d42018-12-03 10:20:33 -05005863 u64 num_bytes = btrfs_calc_trans_metadata_size(fs_info, nr);
5864 u64 released = 0;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04005865
Josef Bacikba2c4d42018-12-03 10:20:33 -05005866 released = block_rsv_release_bytes(fs_info, block_rsv, global_rsv,
5867 num_bytes, NULL);
5868 if (released)
5869 trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
5870 0, released, 0);
Yan, Zhengf0486c62010-05-16 10:46:25 -04005871}
5872
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005873static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
5874{
5875 struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
5876 struct btrfs_space_info *sinfo = block_rsv->space_info;
5877 u64 num_bytes;
5878
Josef Bacikae2e4722016-05-27 12:58:35 -04005879 /*
5880 * The global block rsv is based on the size of the extent tree, the
5881 * checksum tree and the root tree. If the fs is empty we want to set
5882 * it to a minimal amount for safety.
5883 */
5884 num_bytes = btrfs_root_used(&fs_info->extent_root->root_item) +
5885 btrfs_root_used(&fs_info->csum_root->root_item) +
5886 btrfs_root_used(&fs_info->tree_root->root_item);
5887 num_bytes = max_t(u64, num_bytes, SZ_16M);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005888
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005889 spin_lock(&sinfo->lock);
Stefan Behrens1f699d32012-04-27 12:41:46 -04005890 spin_lock(&block_rsv->lock);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005891
Byongho Leeee221842015-12-15 01:42:10 +09005892 block_rsv->size = min_t(u64, num_bytes, SZ_512M);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005893
Josef Bacikfb4b10e2016-01-11 17:28:38 -05005894 if (block_rsv->reserved < block_rsv->size) {
Liu Bo41361352017-02-13 15:42:21 -08005895 num_bytes = btrfs_space_info_used(sinfo, true);
Josef Bacikfb4b10e2016-01-11 17:28:38 -05005896 if (sinfo->total_bytes > num_bytes) {
5897 num_bytes = sinfo->total_bytes - num_bytes;
5898 num_bytes = min(num_bytes,
5899 block_rsv->size - block_rsv->reserved);
5900 block_rsv->reserved += num_bytes;
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08005901 update_bytes_may_use(sinfo, num_bytes);
Josef Bacikfb4b10e2016-01-11 17:28:38 -05005902 trace_btrfs_space_reservation(fs_info, "space_info",
5903 sinfo->flags, num_bytes,
5904 1);
5905 }
5906 } else if (block_rsv->reserved > block_rsv->size) {
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005907 num_bytes = block_rsv->reserved - block_rsv->size;
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08005908 update_bytes_may_use(sinfo, -num_bytes);
Josef Bacik8c2a3ca2012-01-10 10:31:31 -05005909 trace_btrfs_space_reservation(fs_info, "space_info",
Liu Bo2bcc0322012-03-29 09:57:44 -04005910 sinfo->flags, num_bytes, 0);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005911 block_rsv->reserved = block_rsv->size;
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005912 }
David Sterba182608c2011-05-05 13:13:16 +02005913
Josef Bacikfb4b10e2016-01-11 17:28:38 -05005914 if (block_rsv->reserved == block_rsv->size)
5915 block_rsv->full = 1;
5916 else
5917 block_rsv->full = 0;
5918
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005919 spin_unlock(&block_rsv->lock);
Stefan Behrens1f699d32012-04-27 12:41:46 -04005920 spin_unlock(&sinfo->lock);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005921}
5922
Yan, Zhengf0486c62010-05-16 10:46:25 -04005923static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
5924{
5925 struct btrfs_space_info *space_info;
5926
5927 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
5928 fs_info->chunk_block_rsv.space_info = space_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005929
5930 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005931 fs_info->global_block_rsv.space_info = space_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005932 fs_info->trans_block_rsv.space_info = space_info;
5933 fs_info->empty_block_rsv.space_info = space_info;
Josef Bacik6d668dd2011-11-03 22:54:25 -04005934 fs_info->delayed_block_rsv.space_info = space_info;
Josef Bacikba2c4d42018-12-03 10:20:33 -05005935 fs_info->delayed_refs_rsv.space_info = space_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005936
Josef Bacikba2c4d42018-12-03 10:20:33 -05005937 fs_info->extent_root->block_rsv = &fs_info->delayed_refs_rsv;
5938 fs_info->csum_root->block_rsv = &fs_info->delayed_refs_rsv;
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005939 fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
5940 fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
Stefan Behrens3a6cad92013-05-16 14:48:19 +00005941 if (fs_info->quota_root)
5942 fs_info->quota_root->block_rsv = &fs_info->global_block_rsv;
Yan, Zhengf0486c62010-05-16 10:46:25 -04005943 fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005944
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005945 update_global_block_rsv(fs_info);
5946}
5947
5948static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
5949{
Josef Bacik8c2a3ca2012-01-10 10:31:31 -05005950 block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
Qu Wenruoff6bc372017-12-21 13:42:04 +08005951 (u64)-1, NULL);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04005952 WARN_ON(fs_info->trans_block_rsv.size > 0);
5953 WARN_ON(fs_info->trans_block_rsv.reserved > 0);
5954 WARN_ON(fs_info->chunk_block_rsv.size > 0);
5955 WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
Josef Bacik6d668dd2011-11-03 22:54:25 -04005956 WARN_ON(fs_info->delayed_block_rsv.size > 0);
5957 WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
Josef Bacikba2c4d42018-12-03 10:20:33 -05005958 WARN_ON(fs_info->delayed_refs_rsv.reserved > 0);
5959 WARN_ON(fs_info->delayed_refs_rsv.size > 0);
Josef Bacikfcb80c22011-05-03 10:40:22 -04005960}
5961
Josef Bacikba2c4d42018-12-03 10:20:33 -05005962/*
5963 * btrfs_update_delayed_refs_rsv - adjust the size of the delayed refs rsv
5964 * @trans - the trans that may have generated delayed refs
5965 *
5966 * This is to be called anytime we may have adjusted trans->delayed_ref_updates,
5967 * it'll calculate the additional size and add it to the delayed_refs_rsv.
5968 */
5969void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans)
5970{
5971 struct btrfs_fs_info *fs_info = trans->fs_info;
5972 struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv;
5973 u64 num_bytes;
5974
5975 if (!trans->delayed_ref_updates)
5976 return;
5977
5978 num_bytes = btrfs_calc_trans_metadata_size(fs_info,
5979 trans->delayed_ref_updates);
5980 spin_lock(&delayed_rsv->lock);
5981 delayed_rsv->size += num_bytes;
5982 delayed_rsv->full = 0;
5983 spin_unlock(&delayed_rsv->lock);
5984 trans->delayed_ref_updates = 0;
5985}
Yan, Zhenga22285a2010-05-16 10:48:46 -04005986
Filipe Manana4fbcdf62015-05-20 14:01:54 +01005987/*
5988 * To be called after all the new block groups attached to the transaction
5989 * handle have been created (btrfs_create_pending_block_groups()).
5990 */
5991void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans)
5992{
Jeff Mahoney64b63582016-06-20 17:23:41 -04005993 struct btrfs_fs_info *fs_info = trans->fs_info;
Filipe Manana4fbcdf62015-05-20 14:01:54 +01005994
5995 if (!trans->chunk_bytes_reserved)
5996 return;
5997
5998 WARN_ON_ONCE(!list_empty(&trans->new_bgs));
5999
6000 block_rsv_release_bytes(fs_info, &fs_info->chunk_block_rsv, NULL,
Qu Wenruoff6bc372017-12-21 13:42:04 +08006001 trans->chunk_bytes_reserved, NULL);
Filipe Manana4fbcdf62015-05-20 14:01:54 +01006002 trans->chunk_bytes_reserved = 0;
6003}
6004
Miao Xied5c12072013-02-28 10:04:33 +00006005/*
6006 * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation
6007 * root: the root of the parent directory
6008 * rsv: block reservation
6009 * items: the number of items that we need do reservation
Lu Fengqia5b7f422018-08-09 09:46:04 +08006010 * use_global_rsv: allow fallback to the global block reservation
Miao Xied5c12072013-02-28 10:04:33 +00006011 *
6012 * This function is used to reserve the space for snapshot/subvolume
6013 * creation and deletion. Those operations are different with the
6014 * common file/directory operations, they change two fs/file trees
6015 * and root tree, the number of items that the qgroup reserves is
6016 * different with the free space reservation. So we can not use
Nicholas D Steeves01327612016-05-19 21:18:45 -04006017 * the space reservation mechanism in start_transaction().
Miao Xied5c12072013-02-28 10:04:33 +00006018 */
6019int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
Lu Fengqia5b7f422018-08-09 09:46:04 +08006020 struct btrfs_block_rsv *rsv, int items,
Jeff Mahoneyee3441b2013-07-09 16:37:21 -04006021 bool use_global_rsv)
Yan, Zhenga22285a2010-05-16 10:48:46 -04006022{
Lu Fengqia5b7f422018-08-09 09:46:04 +08006023 u64 qgroup_num_bytes = 0;
Miao Xied5c12072013-02-28 10:04:33 +00006024 u64 num_bytes;
6025 int ret;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006026 struct btrfs_fs_info *fs_info = root->fs_info;
6027 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
Miao Xied5c12072013-02-28 10:04:33 +00006028
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006029 if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) {
Miao Xied5c12072013-02-28 10:04:33 +00006030 /* One for parent inode, two for dir entries */
Lu Fengqia5b7f422018-08-09 09:46:04 +08006031 qgroup_num_bytes = 3 * fs_info->nodesize;
6032 ret = btrfs_qgroup_reserve_meta_prealloc(root,
6033 qgroup_num_bytes, true);
Miao Xied5c12072013-02-28 10:04:33 +00006034 if (ret)
6035 return ret;
Miao Xied5c12072013-02-28 10:04:33 +00006036 }
6037
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006038 num_bytes = btrfs_calc_trans_metadata_size(fs_info, items);
6039 rsv->space_info = __find_space_info(fs_info,
Miao Xied5c12072013-02-28 10:04:33 +00006040 BTRFS_BLOCK_GROUP_METADATA);
6041 ret = btrfs_block_rsv_add(root, rsv, num_bytes,
6042 BTRFS_RESERVE_FLUSH_ALL);
Jeff Mahoneyee3441b2013-07-09 16:37:21 -04006043
6044 if (ret == -ENOSPC && use_global_rsv)
Lu Fengqi3a584172018-08-04 21:10:55 +08006045 ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes, true);
Jeff Mahoneyee3441b2013-07-09 16:37:21 -04006046
Lu Fengqia5b7f422018-08-09 09:46:04 +08006047 if (ret && qgroup_num_bytes)
6048 btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes);
Miao Xied5c12072013-02-28 10:04:33 +00006049
6050 return ret;
6051}
6052
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006053void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info,
David Sterba7775c812017-02-10 19:18:18 +01006054 struct btrfs_block_rsv *rsv)
Miao Xied5c12072013-02-28 10:04:33 +00006055{
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006056 btrfs_block_rsv_release(fs_info, rsv, (u64)-1);
Yan, Zhenga22285a2010-05-16 10:48:46 -04006057}
6058
Josef Bacik69fe2d72017-10-19 14:15:57 -04006059static void btrfs_calculate_inode_block_rsv_size(struct btrfs_fs_info *fs_info,
6060 struct btrfs_inode *inode)
Josef Bacik9e0baf62011-07-15 15:16:44 +00006061{
Josef Bacik69fe2d72017-10-19 14:15:57 -04006062 struct btrfs_block_rsv *block_rsv = &inode->block_rsv;
6063 u64 reserve_size = 0;
Qu Wenruoff6bc372017-12-21 13:42:04 +08006064 u64 qgroup_rsv_size = 0;
Josef Bacik69fe2d72017-10-19 14:15:57 -04006065 u64 csum_leaves;
6066 unsigned outstanding_extents;
Josef Bacik9e0baf62011-07-15 15:16:44 +00006067
Josef Bacik69fe2d72017-10-19 14:15:57 -04006068 lockdep_assert_held(&inode->lock);
6069 outstanding_extents = inode->outstanding_extents;
6070 if (outstanding_extents)
6071 reserve_size = btrfs_calc_trans_metadata_size(fs_info,
6072 outstanding_extents + 1);
6073 csum_leaves = btrfs_csum_bytes_to_leaves(fs_info,
6074 inode->csum_bytes);
6075 reserve_size += btrfs_calc_trans_metadata_size(fs_info,
6076 csum_leaves);
Qu Wenruoff6bc372017-12-21 13:42:04 +08006077 /*
6078 * For qgroup rsv, the calculation is very simple:
6079 * account one nodesize for each outstanding extent
6080 *
6081 * This is overestimating in most cases.
6082 */
Nikolay Borisov139a5612019-03-18 17:45:20 +02006083 qgroup_rsv_size = (u64)outstanding_extents * fs_info->nodesize;
Josef Bacik9e0baf62011-07-15 15:16:44 +00006084
Josef Bacik69fe2d72017-10-19 14:15:57 -04006085 spin_lock(&block_rsv->lock);
6086 block_rsv->size = reserve_size;
Qu Wenruoff6bc372017-12-21 13:42:04 +08006087 block_rsv->qgroup_rsv_size = qgroup_rsv_size;
Josef Bacik69fe2d72017-10-19 14:15:57 -04006088 spin_unlock(&block_rsv->lock);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006089}
6090
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006091int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes)
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006092{
David Sterba3ffbd682018-06-29 10:56:42 +02006093 struct btrfs_fs_info *fs_info = inode->root->fs_info;
Josef Bacik69fe2d72017-10-19 14:15:57 -04006094 unsigned nr_extents;
Miao Xie08e007d2012-10-16 11:33:38 +00006095 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
Jan Schmidteb6b88d2013-01-27 23:26:00 -07006096 int ret = 0;
Josef Bacikc64c2bd2012-12-14 13:48:14 -05006097 bool delalloc_lock = true;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006098
Josef Bacikc64c2bd2012-12-14 13:48:14 -05006099 /* If we are a free space inode we need to not flush since we will be in
6100 * the middle of a transaction commit. We also don't need the delalloc
6101 * mutex since we won't race with anybody. We need this mostly to make
6102 * lockdep shut its filthy mouth.
Josef Bacikbac357dc2016-07-20 16:48:45 -07006103 *
6104 * If we have a transaction open (can happen if we call truncate_block
6105 * from truncate), then we need FLUSH_LIMIT so we don't deadlock.
Josef Bacikc64c2bd2012-12-14 13:48:14 -05006106 */
6107 if (btrfs_is_free_space_inode(inode)) {
Miao Xie08e007d2012-10-16 11:33:38 +00006108 flush = BTRFS_RESERVE_NO_FLUSH;
Josef Bacikc64c2bd2012-12-14 13:48:14 -05006109 delalloc_lock = false;
Nikolay Borisovda07d4a2018-01-12 16:21:05 +02006110 } else {
6111 if (current->journal_info)
6112 flush = BTRFS_RESERVE_FLUSH_LIMIT;
Josef Bacikc09544e2011-08-30 10:19:10 -04006113
Nikolay Borisovda07d4a2018-01-12 16:21:05 +02006114 if (btrfs_transaction_in_commit(fs_info))
6115 schedule_timeout(1);
6116 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006117
Josef Bacikc64c2bd2012-12-14 13:48:14 -05006118 if (delalloc_lock)
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006119 mutex_lock(&inode->delalloc_mutex);
Josef Bacikc64c2bd2012-12-14 13:48:14 -05006120
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006121 num_bytes = ALIGN(num_bytes, fs_info->sectorsize);
Josef Bacik69fe2d72017-10-19 14:15:57 -04006122
6123 /* Add our new extents and calculate the new rsv size. */
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006124 spin_lock(&inode->lock);
Josef Bacik69fe2d72017-10-19 14:15:57 -04006125 nr_extents = count_max_extents(num_bytes);
Josef Bacik8b62f872017-10-19 14:15:55 -04006126 btrfs_mod_outstanding_extents(inode, nr_extents);
Josef Bacik69fe2d72017-10-19 14:15:57 -04006127 inode->csum_bytes += num_bytes;
6128 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006129 spin_unlock(&inode->lock);
Josef Bacik57a45ced2011-01-25 16:30:38 -05006130
Josef Bacik69fe2d72017-10-19 14:15:57 -04006131 ret = btrfs_inode_rsv_refill(inode, flush);
Qu Wenruo43b18592017-12-12 15:34:32 +08006132 if (unlikely(ret))
Wang Shilong88e081bf2013-03-01 11:36:01 +00006133 goto out_fail;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006134
Josef Bacikc64c2bd2012-12-14 13:48:14 -05006135 if (delalloc_lock)
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006136 mutex_unlock(&inode->delalloc_mutex);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006137 return 0;
Wang Shilong88e081bf2013-03-01 11:36:01 +00006138
6139out_fail:
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006140 spin_lock(&inode->lock);
Josef Bacik8b62f872017-10-19 14:15:55 -04006141 nr_extents = count_max_extents(num_bytes);
6142 btrfs_mod_outstanding_extents(inode, -nr_extents);
Josef Bacik69fe2d72017-10-19 14:15:57 -04006143 inode->csum_bytes -= num_bytes;
6144 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006145 spin_unlock(&inode->lock);
Wang Shilong88e081bf2013-03-01 11:36:01 +00006146
Qu Wenruo43b18592017-12-12 15:34:32 +08006147 btrfs_inode_rsv_release(inode, true);
Wang Shilong88e081bf2013-03-01 11:36:01 +00006148 if (delalloc_lock)
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006149 mutex_unlock(&inode->delalloc_mutex);
Wang Shilong88e081bf2013-03-01 11:36:01 +00006150 return ret;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006151}
6152
Josef Bacik7709cde2011-08-04 10:25:02 -04006153/**
6154 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
Josef Bacik8b62f872017-10-19 14:15:55 -04006155 * @inode: the inode to release the reservation for.
6156 * @num_bytes: the number of bytes we are releasing.
Qu Wenruo43b18592017-12-12 15:34:32 +08006157 * @qgroup_free: free qgroup reservation or convert it to per-trans reservation
Josef Bacik7709cde2011-08-04 10:25:02 -04006158 *
6159 * This will release the metadata reservation for an inode. This can be called
6160 * once we complete IO for a given set of bytes to release their metadata
Josef Bacik8b62f872017-10-19 14:15:55 -04006161 * reservations, or on error for the same reason.
Josef Bacik7709cde2011-08-04 10:25:02 -04006162 */
Qu Wenruo43b18592017-12-12 15:34:32 +08006163void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes,
6164 bool qgroup_free)
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006165{
David Sterba3ffbd682018-06-29 10:56:42 +02006166 struct btrfs_fs_info *fs_info = inode->root->fs_info;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006167
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006168 num_bytes = ALIGN(num_bytes, fs_info->sectorsize);
Nikolay Borisov691fa052017-02-20 13:50:42 +02006169 spin_lock(&inode->lock);
Josef Bacik69fe2d72017-10-19 14:15:57 -04006170 inode->csum_bytes -= num_bytes;
6171 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
Nikolay Borisov691fa052017-02-20 13:50:42 +02006172 spin_unlock(&inode->lock);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006173
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006174 if (btrfs_is_testing(fs_info))
Josef Bacik6a3891c2015-03-16 17:38:52 -04006175 return;
6176
Qu Wenruo43b18592017-12-12 15:34:32 +08006177 btrfs_inode_rsv_release(inode, qgroup_free);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006178}
6179
Josef Bacik7709cde2011-08-04 10:25:02 -04006180/**
Josef Bacik8b62f872017-10-19 14:15:55 -04006181 * btrfs_delalloc_release_extents - release our outstanding_extents
6182 * @inode: the inode to balance the reservation for.
6183 * @num_bytes: the number of bytes we originally reserved with
Qu Wenruo43b18592017-12-12 15:34:32 +08006184 * @qgroup_free: do we need to free qgroup meta reservation or convert them.
Josef Bacik8b62f872017-10-19 14:15:55 -04006185 *
6186 * When we reserve space we increase outstanding_extents for the extents we may
6187 * add. Once we've set the range as delalloc or created our ordered extents we
6188 * have outstanding_extents to track the real usage, so we use this to free our
6189 * temporarily tracked outstanding_extents. This _must_ be used in conjunction
6190 * with btrfs_delalloc_reserve_metadata.
6191 */
Qu Wenruo43b18592017-12-12 15:34:32 +08006192void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes,
6193 bool qgroup_free)
Josef Bacik8b62f872017-10-19 14:15:55 -04006194{
David Sterba3ffbd682018-06-29 10:56:42 +02006195 struct btrfs_fs_info *fs_info = inode->root->fs_info;
Josef Bacik8b62f872017-10-19 14:15:55 -04006196 unsigned num_extents;
Josef Bacik8b62f872017-10-19 14:15:55 -04006197
6198 spin_lock(&inode->lock);
6199 num_extents = count_max_extents(num_bytes);
6200 btrfs_mod_outstanding_extents(inode, -num_extents);
Josef Bacik69fe2d72017-10-19 14:15:57 -04006201 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
Josef Bacik8b62f872017-10-19 14:15:55 -04006202 spin_unlock(&inode->lock);
6203
Josef Bacik8b62f872017-10-19 14:15:55 -04006204 if (btrfs_is_testing(fs_info))
6205 return;
6206
Qu Wenruo43b18592017-12-12 15:34:32 +08006207 btrfs_inode_rsv_release(inode, qgroup_free);
Josef Bacik8b62f872017-10-19 14:15:55 -04006208}
6209
6210/**
Qu Wenruo7cf5b972015-09-08 17:25:55 +08006211 * btrfs_delalloc_reserve_space - reserve data and metadata space for
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006212 * delalloc
6213 * @inode: inode we're writing to
6214 * @start: start range we are writing to
6215 * @len: how long the range we are writing to
Qu Wenruo364ecf32017-02-27 15:10:38 +08006216 * @reserved: mandatory parameter, record actually reserved qgroup ranges of
6217 * current reservation.
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006218 *
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006219 * This will do the following things
6220 *
6221 * o reserve space in data space info for num bytes
6222 * and reserve precious corresponding qgroup space
6223 * (Done in check_data_free_space)
6224 *
6225 * o reserve space for metadata space, based on the number of outstanding
6226 * extents and how much csums will be needed
6227 * also reserve metadata space in a per root over-reserve method.
6228 * o add to the inodes->delalloc_bytes
6229 * o add it to the fs_info's delalloc inodes list.
6230 * (Above 3 all done in delalloc_reserve_metadata)
6231 *
6232 * Return 0 for success
6233 * Return <0 for error(-ENOSPC or -EQUOT)
6234 */
Qu Wenruo364ecf32017-02-27 15:10:38 +08006235int btrfs_delalloc_reserve_space(struct inode *inode,
6236 struct extent_changeset **reserved, u64 start, u64 len)
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006237{
6238 int ret;
6239
Qu Wenruo364ecf32017-02-27 15:10:38 +08006240 ret = btrfs_check_data_free_space(inode, reserved, start, len);
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006241 if (ret < 0)
6242 return ret;
Nikolay Borisov9f3db422017-02-20 13:50:41 +02006243 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len);
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006244 if (ret < 0)
Qu Wenruobc42bda2017-02-27 15:10:39 +08006245 btrfs_free_reserved_data_space(inode, *reserved, start, len);
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006246 return ret;
6247}
6248
6249/**
Qu Wenruo7cf5b972015-09-08 17:25:55 +08006250 * btrfs_delalloc_release_space - release data and metadata space for delalloc
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006251 * @inode: inode we're releasing space for
6252 * @start: start position of the space already reserved
6253 * @len: the len of the space already reserved
Josef Bacik8b62f872017-10-19 14:15:55 -04006254 * @release_bytes: the len of the space we consumed or didn't use
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006255 *
6256 * This function will release the metadata space that was not used and will
6257 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
6258 * list if there are no delalloc bytes left.
6259 * Also it will handle the qgroup reserved space.
6260 */
Qu Wenruobc42bda2017-02-27 15:10:39 +08006261void btrfs_delalloc_release_space(struct inode *inode,
Josef Bacik8b62f872017-10-19 14:15:55 -04006262 struct extent_changeset *reserved,
Qu Wenruo43b18592017-12-12 15:34:32 +08006263 u64 start, u64 len, bool qgroup_free)
Qu Wenruo1ada3a62015-09-08 17:25:53 +08006264{
Qu Wenruo43b18592017-12-12 15:34:32 +08006265 btrfs_delalloc_release_metadata(BTRFS_I(inode), len, qgroup_free);
Qu Wenruobc42bda2017-02-27 15:10:39 +08006266 btrfs_free_reserved_data_space(inode, reserved, start, len);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04006267}
6268
Josef Bacikce93ec52014-11-17 15:45:48 -05006269static int update_block_group(struct btrfs_trans_handle *trans,
David Sterba6b279402019-03-20 12:10:15 +01006270 u64 bytenr, u64 num_bytes, int alloc)
Chris Mason9078a3e2007-04-26 16:46:15 -04006271{
David Sterba6b279402019-03-20 12:10:15 +01006272 struct btrfs_fs_info *info = trans->fs_info;
Josef Bacik0af3d002010-06-21 14:48:16 -04006273 struct btrfs_block_group_cache *cache = NULL;
Chris Masondb945352007-10-15 16:15:53 -04006274 u64 total = num_bytes;
Chris Mason9078a3e2007-04-26 16:46:15 -04006275 u64 old_val;
Chris Masondb945352007-10-15 16:15:53 -04006276 u64 byte_in_group;
Josef Bacik0af3d002010-06-21 14:48:16 -04006277 int factor;
Josef Bacikba2c4d42018-12-03 10:20:33 -05006278 int ret = 0;
Chris Mason3e1ad542007-05-07 20:03:49 -04006279
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006280 /* block accounting for super block */
Miao Xieeb73c1b2013-05-15 07:48:22 +00006281 spin_lock(&info->delalloc_root_lock);
David Sterba6c417612011-04-13 15:41:04 +02006282 old_val = btrfs_super_bytes_used(info->super_copy);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006283 if (alloc)
6284 old_val += num_bytes;
6285 else
6286 old_val -= num_bytes;
David Sterba6c417612011-04-13 15:41:04 +02006287 btrfs_set_super_bytes_used(info->super_copy, old_val);
Miao Xieeb73c1b2013-05-15 07:48:22 +00006288 spin_unlock(&info->delalloc_root_lock);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006289
Chris Masond3977122009-01-05 21:25:51 -05006290 while (total) {
Chris Masondb945352007-10-15 16:15:53 -04006291 cache = btrfs_lookup_block_group(info, bytenr);
Josef Bacikba2c4d42018-12-03 10:20:33 -05006292 if (!cache) {
6293 ret = -ENOENT;
6294 break;
6295 }
David Sterba46df06b2018-07-13 20:46:30 +02006296 factor = btrfs_bg_type_to_factor(cache->flags);
6297
Josef Bacik9d66e232010-08-25 16:54:15 -04006298 /*
6299 * If this block group has free space cache written out, we
6300 * need to make sure to load it if we are removing space. This
6301 * is because we need the unpinning stage to actually add the
6302 * space back to the block group, otherwise we will leak space.
6303 */
6304 if (!alloc && cache->cached == BTRFS_CACHE_NO)
Liu Bof6373bf2012-12-27 09:01:18 +00006305 cache_block_group(cache, 1);
Josef Bacik0af3d002010-06-21 14:48:16 -04006306
Chris Masondb945352007-10-15 16:15:53 -04006307 byte_in_group = bytenr - cache->key.objectid;
6308 WARN_ON(byte_in_group > cache->key.offset);
Chris Mason9078a3e2007-04-26 16:46:15 -04006309
Josef Bacik25179202008-10-29 14:49:05 -04006310 spin_lock(&cache->space_info->lock);
Chris Masonc286ac42008-07-22 23:06:41 -04006311 spin_lock(&cache->lock);
Josef Bacik0af3d002010-06-21 14:48:16 -04006312
Jeff Mahoney6202df62016-06-22 18:54:22 -04006313 if (btrfs_test_opt(info, SPACE_CACHE) &&
Josef Bacik0af3d002010-06-21 14:48:16 -04006314 cache->disk_cache_state < BTRFS_DC_CLEAR)
6315 cache->disk_cache_state = BTRFS_DC_CLEAR;
6316
Chris Mason9078a3e2007-04-26 16:46:15 -04006317 old_val = btrfs_block_group_used(&cache->item);
Chris Masondb945352007-10-15 16:15:53 -04006318 num_bytes = min(total, cache->key.offset - byte_in_group);
Chris Masoncd1bc462007-04-27 10:08:34 -04006319 if (alloc) {
Chris Masondb945352007-10-15 16:15:53 -04006320 old_val += num_bytes;
Yan Zheng11833d62009-09-11 16:11:19 -04006321 btrfs_set_block_group_used(&cache->item, old_val);
6322 cache->reserved -= num_bytes;
Yan Zheng11833d62009-09-11 16:11:19 -04006323 cache->space_info->bytes_reserved -= num_bytes;
Yan, Zhengb742bb822010-05-16 10:46:24 -04006324 cache->space_info->bytes_used += num_bytes;
6325 cache->space_info->disk_used += num_bytes * factor;
Chris Masonc286ac42008-07-22 23:06:41 -04006326 spin_unlock(&cache->lock);
Josef Bacik25179202008-10-29 14:49:05 -04006327 spin_unlock(&cache->space_info->lock);
Chris Masoncd1bc462007-04-27 10:08:34 -04006328 } else {
Chris Masondb945352007-10-15 16:15:53 -04006329 old_val -= num_bytes;
Filipe Mananaae0ab002014-11-26 15:28:52 +00006330 btrfs_set_block_group_used(&cache->item, old_val);
6331 cache->pinned += num_bytes;
Lu Fengqie2907c12018-10-24 20:24:02 +08006332 update_bytes_pinned(cache->space_info, num_bytes);
Filipe Mananaae0ab002014-11-26 15:28:52 +00006333 cache->space_info->bytes_used -= num_bytes;
6334 cache->space_info->disk_used -= num_bytes * factor;
6335 spin_unlock(&cache->lock);
6336 spin_unlock(&cache->space_info->lock);
Josef Bacik47ab2a62014-09-18 11:20:02 -04006337
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006338 trace_btrfs_space_reservation(info, "pinned",
Josef Bacikc51e7bb2016-03-25 13:25:54 -04006339 cache->space_info->flags,
6340 num_bytes, 1);
Ethan Liendec59fa2018-07-13 16:50:42 +08006341 percpu_counter_add_batch(&cache->space_info->total_bytes_pinned,
6342 num_bytes,
6343 BTRFS_TOTAL_BYTES_PINNED_BATCH);
Filipe Mananaae0ab002014-11-26 15:28:52 +00006344 set_extent_dirty(info->pinned_extents,
6345 bytenr, bytenr + num_bytes - 1,
6346 GFP_NOFS | __GFP_NOFAIL);
Chris Masoncd1bc462007-04-27 10:08:34 -04006347 }
Chris Mason1bbc6212015-04-06 12:46:08 -07006348
6349 spin_lock(&trans->transaction->dirty_bgs_lock);
6350 if (list_empty(&cache->dirty_list)) {
6351 list_add_tail(&cache->dirty_list,
6352 &trans->transaction->dirty_bgs);
Josef Bacikba2c4d42018-12-03 10:20:33 -05006353 trans->delayed_ref_updates++;
Chris Mason1bbc6212015-04-06 12:46:08 -07006354 btrfs_get_block_group(cache);
6355 }
6356 spin_unlock(&trans->transaction->dirty_bgs_lock);
6357
Filipe Manana036a9342015-11-23 15:25:16 +00006358 /*
6359 * No longer have used bytes in this block group, queue it for
6360 * deletion. We do this after adding the block group to the
6361 * dirty list to avoid races between cleaner kthread and space
6362 * cache writeout.
6363 */
Qu Wenruo031f24d2018-05-22 16:43:47 +08006364 if (!alloc && old_val == 0)
6365 btrfs_mark_bg_unused(cache);
Filipe Manana036a9342015-11-23 15:25:16 +00006366
Chris Masonfa9c0d792009-04-03 09:47:43 -04006367 btrfs_put_block_group(cache);
Chris Masondb945352007-10-15 16:15:53 -04006368 total -= num_bytes;
6369 bytenr += num_bytes;
Chris Mason9078a3e2007-04-26 16:46:15 -04006370 }
Josef Bacikba2c4d42018-12-03 10:20:33 -05006371
6372 /* Modified block groups are accounted for in the delayed_refs_rsv. */
6373 btrfs_update_delayed_refs_rsv(trans);
6374 return ret;
Chris Mason9078a3e2007-04-26 16:46:15 -04006375}
Chris Mason6324fbf2008-03-24 15:01:59 -04006376
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006377static u64 first_logical_byte(struct btrfs_fs_info *fs_info, u64 search_start)
Chris Masona061fc82008-05-07 11:43:44 -04006378{
Josef Bacik0f9dd462008-09-23 13:14:11 -04006379 struct btrfs_block_group_cache *cache;
Yan Zhengd2fb3432008-12-11 16:30:39 -05006380 u64 bytenr;
Josef Bacik0f9dd462008-09-23 13:14:11 -04006381
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006382 spin_lock(&fs_info->block_group_cache_lock);
6383 bytenr = fs_info->first_logical_byte;
6384 spin_unlock(&fs_info->block_group_cache_lock);
Liu Boa1897fd2012-12-27 09:01:23 +00006385
6386 if (bytenr < (u64)-1)
6387 return bytenr;
6388
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006389 cache = btrfs_lookup_first_block_group(fs_info, search_start);
Josef Bacik0f9dd462008-09-23 13:14:11 -04006390 if (!cache)
Chris Masona061fc82008-05-07 11:43:44 -04006391 return 0;
Josef Bacik0f9dd462008-09-23 13:14:11 -04006392
Yan Zhengd2fb3432008-12-11 16:30:39 -05006393 bytenr = cache->key.objectid;
Chris Masonfa9c0d792009-04-03 09:47:43 -04006394 btrfs_put_block_group(cache);
Yan Zhengd2fb3432008-12-11 16:30:39 -05006395
6396 return bytenr;
Chris Masona061fc82008-05-07 11:43:44 -04006397}
6398
David Sterbafdf08602019-03-20 12:12:32 +01006399static int pin_down_extent(struct btrfs_block_group_cache *cache,
Yan, Zhengf0486c62010-05-16 10:46:25 -04006400 u64 bytenr, u64 num_bytes, int reserved)
Yan324ae4d2007-11-16 14:57:08 -05006401{
David Sterbafdf08602019-03-20 12:12:32 +01006402 struct btrfs_fs_info *fs_info = cache->fs_info;
6403
Yan Zheng11833d62009-09-11 16:11:19 -04006404 spin_lock(&cache->space_info->lock);
6405 spin_lock(&cache->lock);
6406 cache->pinned += num_bytes;
Lu Fengqie2907c12018-10-24 20:24:02 +08006407 update_bytes_pinned(cache->space_info, num_bytes);
Yan Zheng11833d62009-09-11 16:11:19 -04006408 if (reserved) {
6409 cache->reserved -= num_bytes;
6410 cache->space_info->bytes_reserved -= num_bytes;
Yan324ae4d2007-11-16 14:57:08 -05006411 }
Yan Zheng11833d62009-09-11 16:11:19 -04006412 spin_unlock(&cache->lock);
6413 spin_unlock(&cache->space_info->lock);
6414
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006415 trace_btrfs_space_reservation(fs_info, "pinned",
Josef Bacikc51e7bb2016-03-25 13:25:54 -04006416 cache->space_info->flags, num_bytes, 1);
Ethan Liendec59fa2018-07-13 16:50:42 +08006417 percpu_counter_add_batch(&cache->space_info->total_bytes_pinned,
6418 num_bytes, BTRFS_TOTAL_BYTES_PINNED_BATCH);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006419 set_extent_dirty(fs_info->pinned_extents, bytenr,
Yan, Zhengf0486c62010-05-16 10:46:25 -04006420 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
Yan324ae4d2007-11-16 14:57:08 -05006421 return 0;
6422}
Chris Mason9078a3e2007-04-26 16:46:15 -04006423
Yan, Zhengf0486c62010-05-16 10:46:25 -04006424/*
6425 * this function must be called within transaction
6426 */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006427int btrfs_pin_extent(struct btrfs_fs_info *fs_info,
Yan, Zhengf0486c62010-05-16 10:46:25 -04006428 u64 bytenr, u64 num_bytes, int reserved)
Zheng Yane8569812008-09-26 10:05:48 -04006429{
Yan, Zhengf0486c62010-05-16 10:46:25 -04006430 struct btrfs_block_group_cache *cache;
6431
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006432 cache = btrfs_lookup_block_group(fs_info, bytenr);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01006433 BUG_ON(!cache); /* Logic error */
Yan, Zhengf0486c62010-05-16 10:46:25 -04006434
David Sterbafdf08602019-03-20 12:12:32 +01006435 pin_down_extent(cache, bytenr, num_bytes, reserved);
Yan, Zhengf0486c62010-05-16 10:46:25 -04006436
6437 btrfs_put_block_group(cache);
Yan Zheng11833d62009-09-11 16:11:19 -04006438 return 0;
6439}
Zheng Yane8569812008-09-26 10:05:48 -04006440
Yan, Zhengf0486c62010-05-16 10:46:25 -04006441/*
Chris Masone688b7252011-10-31 20:52:39 -04006442 * this function must be called within transaction
Yan, Zhengf0486c62010-05-16 10:46:25 -04006443 */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006444int btrfs_pin_extent_for_log_replay(struct btrfs_fs_info *fs_info,
Chris Masone688b7252011-10-31 20:52:39 -04006445 u64 bytenr, u64 num_bytes)
Yan, Zhengf0486c62010-05-16 10:46:25 -04006446{
Chris Masone688b7252011-10-31 20:52:39 -04006447 struct btrfs_block_group_cache *cache;
Josef Bacikb50c6e22013-04-25 15:55:30 -04006448 int ret;
Chris Masone688b7252011-10-31 20:52:39 -04006449
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006450 cache = btrfs_lookup_block_group(fs_info, bytenr);
Josef Bacikb50c6e22013-04-25 15:55:30 -04006451 if (!cache)
6452 return -EINVAL;
Chris Masone688b7252011-10-31 20:52:39 -04006453
6454 /*
6455 * pull in the free space cache (if any) so that our pin
6456 * removes the free space from the cache. We have load_only set
6457 * to one because the slow code to read in the free extents does check
6458 * the pinned extents.
6459 */
Liu Bof6373bf2012-12-27 09:01:18 +00006460 cache_block_group(cache, 1);
Chris Masone688b7252011-10-31 20:52:39 -04006461
David Sterbafdf08602019-03-20 12:12:32 +01006462 pin_down_extent(cache, bytenr, num_bytes, 0);
Chris Masone688b7252011-10-31 20:52:39 -04006463
6464 /* remove us from the free space cache (if we're there at all) */
Josef Bacikb50c6e22013-04-25 15:55:30 -04006465 ret = btrfs_remove_free_space(cache, bytenr, num_bytes);
Chris Masone688b7252011-10-31 20:52:39 -04006466 btrfs_put_block_group(cache);
Josef Bacikb50c6e22013-04-25 15:55:30 -04006467 return ret;
Chris Masone688b7252011-10-31 20:52:39 -04006468}
6469
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006470static int __exclude_logged_extent(struct btrfs_fs_info *fs_info,
6471 u64 start, u64 num_bytes)
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006472{
6473 int ret;
6474 struct btrfs_block_group_cache *block_group;
6475 struct btrfs_caching_control *caching_ctl;
6476
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006477 block_group = btrfs_lookup_block_group(fs_info, start);
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006478 if (!block_group)
6479 return -EINVAL;
6480
6481 cache_block_group(block_group, 0);
6482 caching_ctl = get_caching_control(block_group);
6483
6484 if (!caching_ctl) {
6485 /* Logic error */
6486 BUG_ON(!block_group_cache_done(block_group));
6487 ret = btrfs_remove_free_space(block_group, start, num_bytes);
6488 } else {
6489 mutex_lock(&caching_ctl->mutex);
6490
6491 if (start >= caching_ctl->progress) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006492 ret = add_excluded_extent(fs_info, start, num_bytes);
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006493 } else if (start + num_bytes <= caching_ctl->progress) {
6494 ret = btrfs_remove_free_space(block_group,
6495 start, num_bytes);
6496 } else {
6497 num_bytes = caching_ctl->progress - start;
6498 ret = btrfs_remove_free_space(block_group,
6499 start, num_bytes);
6500 if (ret)
6501 goto out_lock;
6502
6503 num_bytes = (start + num_bytes) -
6504 caching_ctl->progress;
6505 start = caching_ctl->progress;
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006506 ret = add_excluded_extent(fs_info, start, num_bytes);
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006507 }
6508out_lock:
6509 mutex_unlock(&caching_ctl->mutex);
6510 put_caching_control(caching_ctl);
6511 }
6512 btrfs_put_block_group(block_group);
6513 return ret;
6514}
6515
David Sterbabcdc4282019-03-20 12:14:33 +01006516int btrfs_exclude_logged_extents(struct extent_buffer *eb)
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006517{
David Sterbabcdc4282019-03-20 12:14:33 +01006518 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006519 struct btrfs_file_extent_item *item;
6520 struct btrfs_key key;
6521 int found_type;
6522 int i;
Gu Jinxiangb89311e2018-05-22 17:46:51 +08006523 int ret = 0;
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006524
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006525 if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS))
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006526 return 0;
6527
6528 for (i = 0; i < btrfs_header_nritems(eb); i++) {
6529 btrfs_item_key_to_cpu(eb, &key, i);
6530 if (key.type != BTRFS_EXTENT_DATA_KEY)
6531 continue;
6532 item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
6533 found_type = btrfs_file_extent_type(eb, item);
6534 if (found_type == BTRFS_FILE_EXTENT_INLINE)
6535 continue;
6536 if (btrfs_file_extent_disk_bytenr(eb, item) == 0)
6537 continue;
6538 key.objectid = btrfs_file_extent_disk_bytenr(eb, item);
6539 key.offset = btrfs_file_extent_disk_num_bytes(eb, item);
Gu Jinxiangb89311e2018-05-22 17:46:51 +08006540 ret = __exclude_logged_extent(fs_info, key.objectid, key.offset);
6541 if (ret)
6542 break;
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006543 }
6544
Gu Jinxiangb89311e2018-05-22 17:46:51 +08006545 return ret;
Josef Bacik8c2a1a32013-06-06 13:19:32 -04006546}
6547
Filipe Manana9cfa3e32016-04-26 15:39:32 +01006548static void
6549btrfs_inc_block_group_reservations(struct btrfs_block_group_cache *bg)
6550{
6551 atomic_inc(&bg->reservations);
6552}
6553
6554void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info,
6555 const u64 start)
6556{
6557 struct btrfs_block_group_cache *bg;
6558
6559 bg = btrfs_lookup_block_group(fs_info, start);
6560 ASSERT(bg);
6561 if (atomic_dec_and_test(&bg->reservations))
Peter Zijlstra46259562018-03-15 11:43:08 +01006562 wake_up_var(&bg->reservations);
Filipe Manana9cfa3e32016-04-26 15:39:32 +01006563 btrfs_put_block_group(bg);
6564}
6565
Filipe Manana9cfa3e32016-04-26 15:39:32 +01006566void btrfs_wait_block_group_reservations(struct btrfs_block_group_cache *bg)
6567{
6568 struct btrfs_space_info *space_info = bg->space_info;
6569
6570 ASSERT(bg->ro);
6571
6572 if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA))
6573 return;
6574
6575 /*
6576 * Our block group is read only but before we set it to read only,
6577 * some task might have had allocated an extent from it already, but it
6578 * has not yet created a respective ordered extent (and added it to a
6579 * root's list of ordered extents).
6580 * Therefore wait for any task currently allocating extents, since the
6581 * block group's reservations counter is incremented while a read lock
6582 * on the groups' semaphore is held and decremented after releasing
6583 * the read access on that semaphore and creating the ordered extent.
6584 */
6585 down_write(&space_info->groups_sem);
6586 up_write(&space_info->groups_sem);
6587
Peter Zijlstra46259562018-03-15 11:43:08 +01006588 wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
Filipe Manana9cfa3e32016-04-26 15:39:32 +01006589}
6590
Josef Bacikfb25e912011-07-26 17:00:46 -04006591/**
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006592 * btrfs_add_reserved_bytes - update the block_group and space info counters
Josef Bacikfb25e912011-07-26 17:00:46 -04006593 * @cache: The cache we are manipulating
Wang Xiaoguang18513092016-07-25 15:51:40 +08006594 * @ram_bytes: The number of bytes of file content, and will be same to
6595 * @num_bytes except for the compress path.
Josef Bacikfb25e912011-07-26 17:00:46 -04006596 * @num_bytes: The number of bytes in question
Miao Xiee570fd22014-06-19 10:42:50 +08006597 * @delalloc: The blocks are allocated for the delalloc write
Josef Bacikfb25e912011-07-26 17:00:46 -04006598 *
Xiaoguang Wang745699e2016-09-23 12:38:50 +08006599 * This is called by the allocator when it reserves space. If this is a
6600 * reservation and the block group has become read only we cannot make the
6601 * reservation and return -EAGAIN, otherwise this function always succeeds.
Yan, Zhengf0486c62010-05-16 10:46:25 -04006602 */
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006603static int btrfs_add_reserved_bytes(struct btrfs_block_group_cache *cache,
Wang Xiaoguang18513092016-07-25 15:51:40 +08006604 u64 ram_bytes, u64 num_bytes, int delalloc)
Yan, Zhengf0486c62010-05-16 10:46:25 -04006605{
Josef Bacikfb25e912011-07-26 17:00:46 -04006606 struct btrfs_space_info *space_info = cache->space_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04006607 int ret = 0;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01006608
Josef Bacikfb25e912011-07-26 17:00:46 -04006609 spin_lock(&space_info->lock);
6610 spin_lock(&cache->lock);
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006611 if (cache->ro) {
6612 ret = -EAGAIN;
Josef Bacikfb25e912011-07-26 17:00:46 -04006613 } else {
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006614 cache->reserved += num_bytes;
6615 space_info->bytes_reserved += num_bytes;
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08006616 update_bytes_may_use(space_info, -ram_bytes);
Miao Xiee570fd22014-06-19 10:42:50 +08006617 if (delalloc)
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006618 cache->delalloc_bytes += num_bytes;
Yan, Zhengf0486c62010-05-16 10:46:25 -04006619 }
Josef Bacikfb25e912011-07-26 17:00:46 -04006620 spin_unlock(&cache->lock);
6621 spin_unlock(&space_info->lock);
Yan, Zhengf0486c62010-05-16 10:46:25 -04006622 return ret;
6623}
6624
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006625/**
6626 * btrfs_free_reserved_bytes - update the block_group and space info counters
6627 * @cache: The cache we are manipulating
6628 * @num_bytes: The number of bytes in question
6629 * @delalloc: The blocks are allocated for the delalloc write
6630 *
6631 * This is called by somebody who is freeing space that was never actually used
6632 * on disk. For example if you reserve some space for a new leaf in transaction
6633 * A and before transaction A commits you free that leaf, you call this with
6634 * reserve set to 0 in order to clear the reservation.
6635 */
6636
zhong jiang556f3ca2018-08-17 00:37:14 +08006637static void btrfs_free_reserved_bytes(struct btrfs_block_group_cache *cache,
6638 u64 num_bytes, int delalloc)
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006639{
6640 struct btrfs_space_info *space_info = cache->space_info;
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006641
6642 spin_lock(&space_info->lock);
6643 spin_lock(&cache->lock);
6644 if (cache->ro)
6645 space_info->bytes_readonly += num_bytes;
6646 cache->reserved -= num_bytes;
6647 space_info->bytes_reserved -= num_bytes;
Josef Bacik21a94f72018-10-11 15:54:03 -04006648 space_info->max_extent_size = 0;
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006649
6650 if (delalloc)
6651 cache->delalloc_bytes -= num_bytes;
6652 spin_unlock(&cache->lock);
6653 spin_unlock(&space_info->lock);
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08006654}
David Sterba8b74c032017-02-10 19:20:56 +01006655void btrfs_prepare_extent_commit(struct btrfs_fs_info *fs_info)
Yan Zheng11833d62009-09-11 16:11:19 -04006656{
Yan Zheng11833d62009-09-11 16:11:19 -04006657 struct btrfs_caching_control *next;
6658 struct btrfs_caching_control *caching_ctl;
6659 struct btrfs_block_group_cache *cache;
6660
Josef Bacik9e351cc2014-03-13 15:42:13 -04006661 down_write(&fs_info->commit_root_sem);
Yan Zheng11833d62009-09-11 16:11:19 -04006662
6663 list_for_each_entry_safe(caching_ctl, next,
6664 &fs_info->caching_block_groups, list) {
6665 cache = caching_ctl->block_group;
6666 if (block_group_cache_done(cache)) {
6667 cache->last_byte_to_unpin = (u64)-1;
6668 list_del_init(&caching_ctl->list);
6669 put_caching_control(caching_ctl);
6670 } else {
6671 cache->last_byte_to_unpin = caching_ctl->progress;
6672 }
6673 }
6674
6675 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
6676 fs_info->pinned_extents = &fs_info->freed_extents[1];
6677 else
6678 fs_info->pinned_extents = &fs_info->freed_extents[0];
6679
Josef Bacik9e351cc2014-03-13 15:42:13 -04006680 up_write(&fs_info->commit_root_sem);
Yan, Zheng8929ecfa2010-05-16 10:49:58 -04006681
6682 update_global_block_rsv(fs_info);
Yan Zheng11833d62009-09-11 16:11:19 -04006683}
6684
Josef Bacikc759c4e2015-10-02 15:25:10 -04006685/*
6686 * Returns the free cluster for the given space info and sets empty_cluster to
6687 * what it should be based on the mount options.
6688 */
6689static struct btrfs_free_cluster *
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006690fetch_cluster_info(struct btrfs_fs_info *fs_info,
6691 struct btrfs_space_info *space_info, u64 *empty_cluster)
Josef Bacikc759c4e2015-10-02 15:25:10 -04006692{
6693 struct btrfs_free_cluster *ret = NULL;
Josef Bacikc759c4e2015-10-02 15:25:10 -04006694
6695 *empty_cluster = 0;
6696 if (btrfs_mixed_space_info(space_info))
6697 return ret;
6698
Josef Bacikc759c4e2015-10-02 15:25:10 -04006699 if (space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006700 ret = &fs_info->meta_alloc_cluster;
Hans van Kranenburg583b7232017-07-28 08:31:28 +02006701 if (btrfs_test_opt(fs_info, SSD))
6702 *empty_cluster = SZ_2M;
6703 else
Byongho Leeee221842015-12-15 01:42:10 +09006704 *empty_cluster = SZ_64K;
Hans van Kranenburg583b7232017-07-28 08:31:28 +02006705 } else if ((space_info->flags & BTRFS_BLOCK_GROUP_DATA) &&
6706 btrfs_test_opt(fs_info, SSD_SPREAD)) {
6707 *empty_cluster = SZ_2M;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006708 ret = &fs_info->data_alloc_cluster;
Josef Bacikc759c4e2015-10-02 15:25:10 -04006709 }
6710
6711 return ret;
6712}
6713
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006714static int unpin_extent_range(struct btrfs_fs_info *fs_info,
6715 u64 start, u64 end,
Filipe Manana678886b2014-12-07 21:31:47 +00006716 const bool return_free_space)
Yan Zheng11833d62009-09-11 16:11:19 -04006717{
Yan Zheng11833d62009-09-11 16:11:19 -04006718 struct btrfs_block_group_cache *cache = NULL;
Josef Bacik7b398f82012-10-22 15:52:28 -04006719 struct btrfs_space_info *space_info;
6720 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
Josef Bacikc759c4e2015-10-02 15:25:10 -04006721 struct btrfs_free_cluster *cluster = NULL;
Yan Zheng11833d62009-09-11 16:11:19 -04006722 u64 len;
Josef Bacikc759c4e2015-10-02 15:25:10 -04006723 u64 total_unpinned = 0;
6724 u64 empty_cluster = 0;
Josef Bacik7b398f82012-10-22 15:52:28 -04006725 bool readonly;
Yan Zheng11833d62009-09-11 16:11:19 -04006726
6727 while (start <= end) {
Josef Bacik7b398f82012-10-22 15:52:28 -04006728 readonly = false;
Yan Zheng11833d62009-09-11 16:11:19 -04006729 if (!cache ||
6730 start >= cache->key.objectid + cache->key.offset) {
6731 if (cache)
6732 btrfs_put_block_group(cache);
Josef Bacikc759c4e2015-10-02 15:25:10 -04006733 total_unpinned = 0;
Yan Zheng11833d62009-09-11 16:11:19 -04006734 cache = btrfs_lookup_block_group(fs_info, start);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01006735 BUG_ON(!cache); /* Logic error */
Josef Bacikc759c4e2015-10-02 15:25:10 -04006736
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006737 cluster = fetch_cluster_info(fs_info,
Josef Bacikc759c4e2015-10-02 15:25:10 -04006738 cache->space_info,
6739 &empty_cluster);
6740 empty_cluster <<= 1;
Yan Zheng11833d62009-09-11 16:11:19 -04006741 }
6742
6743 len = cache->key.objectid + cache->key.offset - start;
6744 len = min(len, end + 1 - start);
6745
6746 if (start < cache->last_byte_to_unpin) {
6747 len = min(len, cache->last_byte_to_unpin - start);
Filipe Manana678886b2014-12-07 21:31:47 +00006748 if (return_free_space)
6749 btrfs_add_free_space(cache, start, len);
Yan Zheng11833d62009-09-11 16:11:19 -04006750 }
Josef Bacik25179202008-10-29 14:49:05 -04006751
Yan, Zhengf0486c62010-05-16 10:46:25 -04006752 start += len;
Josef Bacikc759c4e2015-10-02 15:25:10 -04006753 total_unpinned += len;
Josef Bacik7b398f82012-10-22 15:52:28 -04006754 space_info = cache->space_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04006755
Josef Bacikc759c4e2015-10-02 15:25:10 -04006756 /*
6757 * If this space cluster has been marked as fragmented and we've
6758 * unpinned enough in this block group to potentially allow a
6759 * cluster to be created inside of it go ahead and clear the
6760 * fragmented check.
6761 */
6762 if (cluster && cluster->fragmented &&
6763 total_unpinned > empty_cluster) {
6764 spin_lock(&cluster->lock);
6765 cluster->fragmented = 0;
6766 spin_unlock(&cluster->lock);
6767 }
6768
Josef Bacik7b398f82012-10-22 15:52:28 -04006769 spin_lock(&space_info->lock);
Josef Bacik25179202008-10-29 14:49:05 -04006770 spin_lock(&cache->lock);
Yan Zheng11833d62009-09-11 16:11:19 -04006771 cache->pinned -= len;
Lu Fengqie2907c12018-10-24 20:24:02 +08006772 update_bytes_pinned(space_info, -len);
Josef Bacikc51e7bb2016-03-25 13:25:54 -04006773
6774 trace_btrfs_space_reservation(fs_info, "pinned",
6775 space_info->flags, len, 0);
Josef Bacik4f4db212015-09-29 11:40:47 -04006776 space_info->max_extent_size = 0;
Ethan Liendec59fa2018-07-13 16:50:42 +08006777 percpu_counter_add_batch(&space_info->total_bytes_pinned,
6778 -len, BTRFS_TOTAL_BYTES_PINNED_BATCH);
Josef Bacik7b398f82012-10-22 15:52:28 -04006779 if (cache->ro) {
6780 space_info->bytes_readonly += len;
6781 readonly = true;
6782 }
Josef Bacik25179202008-10-29 14:49:05 -04006783 spin_unlock(&cache->lock);
Josef Bacik957780e2016-05-17 13:30:55 -04006784 if (!readonly && return_free_space &&
6785 global_rsv->space_info == space_info) {
6786 u64 to_add = len;
Nikolay Borisov92ac58e2017-08-17 10:52:28 +03006787
Josef Bacik7b398f82012-10-22 15:52:28 -04006788 spin_lock(&global_rsv->lock);
6789 if (!global_rsv->full) {
Josef Bacik957780e2016-05-17 13:30:55 -04006790 to_add = min(len, global_rsv->size -
6791 global_rsv->reserved);
6792 global_rsv->reserved += to_add;
Qu Wenruo9f9b8e82018-10-24 20:24:01 +08006793 update_bytes_may_use(space_info, to_add);
Josef Bacik7b398f82012-10-22 15:52:28 -04006794 if (global_rsv->reserved >= global_rsv->size)
6795 global_rsv->full = 1;
Josef Bacik957780e2016-05-17 13:30:55 -04006796 trace_btrfs_space_reservation(fs_info,
6797 "space_info",
6798 space_info->flags,
6799 to_add, 1);
6800 len -= to_add;
Josef Bacik7b398f82012-10-22 15:52:28 -04006801 }
6802 spin_unlock(&global_rsv->lock);
Josef Bacik957780e2016-05-17 13:30:55 -04006803 /* Add to any tickets we may have */
6804 if (len)
6805 space_info_add_new_bytes(fs_info, space_info,
6806 len);
Josef Bacik7b398f82012-10-22 15:52:28 -04006807 }
6808 spin_unlock(&space_info->lock);
Yan Zheng11833d62009-09-11 16:11:19 -04006809 }
6810
6811 if (cache)
Chris Masonfa9c0d792009-04-03 09:47:43 -04006812 btrfs_put_block_group(cache);
Chris Masonccd467d2007-06-28 15:57:36 -04006813 return 0;
6814}
6815
Nikolay Borisov5ead2dd2018-03-15 16:00:26 +02006816int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans)
Chris Masona28ec192007-03-06 20:08:01 -05006817{
Nikolay Borisov5ead2dd2018-03-15 16:00:26 +02006818 struct btrfs_fs_info *fs_info = trans->fs_info;
Jeff Mahoneye33e17e2015-06-15 09:41:19 -04006819 struct btrfs_block_group_cache *block_group, *tmp;
6820 struct list_head *deleted_bgs;
Yan Zheng11833d62009-09-11 16:11:19 -04006821 struct extent_io_tree *unpin;
Chris Mason1a5bc162007-10-15 16:15:26 -04006822 u64 start;
6823 u64 end;
Chris Masona28ec192007-03-06 20:08:01 -05006824 int ret;
Chris Masona28ec192007-03-06 20:08:01 -05006825
Yan Zheng11833d62009-09-11 16:11:19 -04006826 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
6827 unpin = &fs_info->freed_extents[1];
6828 else
6829 unpin = &fs_info->freed_extents[0];
6830
Jeff Mahoneye33e17e2015-06-15 09:41:19 -04006831 while (!trans->aborted) {
Filipe Manana0e6ec382018-11-16 13:04:44 +00006832 struct extent_state *cached_state = NULL;
6833
Filipe Mananad4b450c2015-01-29 19:18:25 +00006834 mutex_lock(&fs_info->unused_bg_unpin_mutex);
Chris Mason1a5bc162007-10-15 16:15:26 -04006835 ret = find_first_extent_bit(unpin, 0, &start, &end,
Filipe Manana0e6ec382018-11-16 13:04:44 +00006836 EXTENT_DIRTY, &cached_state);
Filipe Mananad4b450c2015-01-29 19:18:25 +00006837 if (ret) {
6838 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
Chris Masona28ec192007-03-06 20:08:01 -05006839 break;
Filipe Mananad4b450c2015-01-29 19:18:25 +00006840 }
Liu Hui1f3c79a2009-01-05 15:57:51 -05006841
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006842 if (btrfs_test_opt(fs_info, DISCARD))
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006843 ret = btrfs_discard_extent(fs_info, start,
Li Dongyang5378e602011-03-24 10:24:27 +00006844 end + 1 - start, NULL);
Liu Hui1f3c79a2009-01-05 15:57:51 -05006845
Filipe Manana0e6ec382018-11-16 13:04:44 +00006846 clear_extent_dirty(unpin, start, end, &cached_state);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006847 unpin_extent_range(fs_info, start, end, true);
Filipe Mananad4b450c2015-01-29 19:18:25 +00006848 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
Filipe Manana0e6ec382018-11-16 13:04:44 +00006849 free_extent_state(cached_state);
Chris Masonb9473432009-03-13 11:00:37 -04006850 cond_resched();
Chris Masona28ec192007-03-06 20:08:01 -05006851 }
Josef Bacik817d52f2009-07-13 21:29:25 -04006852
Jeff Mahoneye33e17e2015-06-15 09:41:19 -04006853 /*
6854 * Transaction is finished. We don't need the lock anymore. We
6855 * do need to clean up the block groups in case of a transaction
6856 * abort.
6857 */
6858 deleted_bgs = &trans->transaction->deleted_bgs;
6859 list_for_each_entry_safe(block_group, tmp, deleted_bgs, bg_list) {
6860 u64 trimmed = 0;
6861
6862 ret = -EROFS;
6863 if (!trans->aborted)
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04006864 ret = btrfs_discard_extent(fs_info,
Jeff Mahoneye33e17e2015-06-15 09:41:19 -04006865 block_group->key.objectid,
6866 block_group->key.offset,
6867 &trimmed);
6868
6869 list_del_init(&block_group->bg_list);
6870 btrfs_put_block_group_trimming(block_group);
6871 btrfs_put_block_group(block_group);
6872
6873 if (ret) {
6874 const char *errstr = btrfs_decode_error(ret);
6875 btrfs_warn(fs_info,
David Sterba913e1532017-07-13 15:32:18 +02006876 "discard failed while removing blockgroup: errno=%d %s",
Jeff Mahoneye33e17e2015-06-15 09:41:19 -04006877 ret, errstr);
6878 }
6879 }
6880
Chris Masone20d96d2007-03-22 12:13:20 -04006881 return 0;
6882}
6883
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006884static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
Nikolay Borisove72cb922018-06-20 15:48:57 +03006885 struct btrfs_delayed_ref_node *node, u64 parent,
6886 u64 root_objectid, u64 owner_objectid,
6887 u64 owner_offset, int refs_to_drop,
6888 struct btrfs_delayed_extent_op *extent_op)
Chris Masona28ec192007-03-06 20:08:01 -05006889{
Nikolay Borisove72cb922018-06-20 15:48:57 +03006890 struct btrfs_fs_info *info = trans->fs_info;
Chris Masone2fa7222007-03-12 16:22:34 -04006891 struct btrfs_key key;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006892 struct btrfs_path *path;
Chris Mason1261ec42007-03-20 20:35:03 -04006893 struct btrfs_root *extent_root = info->extent_root;
Chris Mason5f39d392007-10-15 16:14:19 -04006894 struct extent_buffer *leaf;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006895 struct btrfs_extent_item *ei;
6896 struct btrfs_extent_inline_ref *iref;
Chris Masona28ec192007-03-06 20:08:01 -05006897 int ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006898 int is_data;
Chris Mason952fcca2008-02-18 16:33:44 -05006899 int extent_slot = 0;
6900 int found_extent = 0;
6901 int num_to_del = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006902 u32 item_size;
6903 u64 refs;
Qu Wenruoc682f9b2015-03-17 16:59:47 +08006904 u64 bytenr = node->bytenr;
6905 u64 num_bytes = node->num_bytes;
Josef Bacikfcebe452014-05-13 17:30:47 -07006906 int last_ref = 0;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04006907 bool skinny_metadata = btrfs_fs_incompat(info, SKINNY_METADATA);
Chris Mason037e6392007-03-07 11:50:24 -05006908
Chris Mason5caf2a02007-04-02 11:20:42 -04006909 path = btrfs_alloc_path();
Chris Mason54aa1f42007-06-22 14:16:25 -04006910 if (!path)
6911 return -ENOMEM;
6912
David Sterbae4058b52015-11-27 16:31:35 +01006913 path->reada = READA_FORWARD;
Chris Masonb9473432009-03-13 11:00:37 -04006914 path->leave_spinning = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006915
6916 is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
6917 BUG_ON(!is_data && refs_to_drop != 1);
6918
Josef Bacik3173a182013-03-07 14:22:04 -05006919 if (is_data)
Thomas Meyer897ca812017-10-07 16:02:21 +02006920 skinny_metadata = false;
Josef Bacik3173a182013-03-07 14:22:04 -05006921
Nikolay Borisovfbe48012018-06-20 15:48:52 +03006922 ret = lookup_extent_backref(trans, path, &iref, bytenr, num_bytes,
6923 parent, root_objectid, owner_objectid,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006924 owner_offset);
Chris Mason7bb86312007-12-11 09:25:06 -05006925 if (ret == 0) {
Chris Mason952fcca2008-02-18 16:33:44 -05006926 extent_slot = path->slots[0];
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006927 while (extent_slot >= 0) {
6928 btrfs_item_key_to_cpu(path->nodes[0], &key,
Chris Mason952fcca2008-02-18 16:33:44 -05006929 extent_slot);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006930 if (key.objectid != bytenr)
Chris Mason952fcca2008-02-18 16:33:44 -05006931 break;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006932 if (key.type == BTRFS_EXTENT_ITEM_KEY &&
6933 key.offset == num_bytes) {
Chris Mason952fcca2008-02-18 16:33:44 -05006934 found_extent = 1;
6935 break;
6936 }
Josef Bacik3173a182013-03-07 14:22:04 -05006937 if (key.type == BTRFS_METADATA_ITEM_KEY &&
6938 key.offset == owner_objectid) {
6939 found_extent = 1;
6940 break;
6941 }
Chris Mason952fcca2008-02-18 16:33:44 -05006942 if (path->slots[0] - extent_slot > 5)
6943 break;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006944 extent_slot--;
Chris Mason952fcca2008-02-18 16:33:44 -05006945 }
Nikolay Borisova79865c2018-06-21 09:45:00 +03006946
Zheng Yan31840ae2008-09-23 13:14:14 -04006947 if (!found_extent) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006948 BUG_ON(iref);
Nikolay Borisov87cc7a82018-06-20 15:49:12 +03006949 ret = remove_extent_backref(trans, path, NULL,
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05006950 refs_to_drop,
Josef Bacikfcebe452014-05-13 17:30:47 -07006951 is_data, &last_ref);
David Sterba005d6422012-09-18 07:52:32 -06006952 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04006953 btrfs_abort_transaction(trans, ret);
David Sterba005d6422012-09-18 07:52:32 -06006954 goto out;
6955 }
David Sterbab3b4aa72011-04-21 01:20:15 +02006956 btrfs_release_path(path);
Chris Masonb9473432009-03-13 11:00:37 -04006957 path->leave_spinning = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04006958
6959 key.objectid = bytenr;
6960 key.type = BTRFS_EXTENT_ITEM_KEY;
6961 key.offset = num_bytes;
6962
Josef Bacik3173a182013-03-07 14:22:04 -05006963 if (!is_data && skinny_metadata) {
6964 key.type = BTRFS_METADATA_ITEM_KEY;
6965 key.offset = owner_objectid;
6966 }
6967
Zheng Yan31840ae2008-09-23 13:14:14 -04006968 ret = btrfs_search_slot(trans, extent_root,
6969 &key, path, -1, 1);
Josef Bacik3173a182013-03-07 14:22:04 -05006970 if (ret > 0 && skinny_metadata && path->slots[0]) {
6971 /*
6972 * Couldn't find our skinny metadata item,
6973 * see if we have ye olde extent item.
6974 */
6975 path->slots[0]--;
6976 btrfs_item_key_to_cpu(path->nodes[0], &key,
6977 path->slots[0]);
6978 if (key.objectid == bytenr &&
6979 key.type == BTRFS_EXTENT_ITEM_KEY &&
6980 key.offset == num_bytes)
6981 ret = 0;
6982 }
6983
6984 if (ret > 0 && skinny_metadata) {
6985 skinny_metadata = false;
Filipe Manana9ce49a02014-04-24 15:15:28 +01006986 key.objectid = bytenr;
Josef Bacik3173a182013-03-07 14:22:04 -05006987 key.type = BTRFS_EXTENT_ITEM_KEY;
6988 key.offset = num_bytes;
6989 btrfs_release_path(path);
6990 ret = btrfs_search_slot(trans, extent_root,
6991 &key, path, -1, 1);
6992 }
6993
Josef Bacikf3465ca2008-11-12 14:19:50 -05006994 if (ret) {
Jeff Mahoney5d163e02016-09-20 10:05:00 -04006995 btrfs_err(info,
6996 "umm, got %d back from search, was looking for %llu",
6997 ret, bytenr);
Josef Bacikb783e622011-07-13 15:03:50 +00006998 if (ret > 0)
David Sterbaa4f78752017-06-29 18:37:49 +02006999 btrfs_print_leaf(path->nodes[0]);
Josef Bacikf3465ca2008-11-12 14:19:50 -05007000 }
David Sterba005d6422012-09-18 07:52:32 -06007001 if (ret < 0) {
Jeff Mahoney66642832016-06-10 18:19:25 -04007002 btrfs_abort_transaction(trans, ret);
David Sterba005d6422012-09-18 07:52:32 -06007003 goto out;
7004 }
Zheng Yan31840ae2008-09-23 13:14:14 -04007005 extent_slot = path->slots[0];
7006 }
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05307007 } else if (WARN_ON(ret == -ENOENT)) {
David Sterbaa4f78752017-06-29 18:37:49 +02007008 btrfs_print_leaf(path->nodes[0]);
Simon Kirbyc2cf52e2013-03-19 22:41:23 +00007009 btrfs_err(info,
7010 "unable to find ref byte nr %llu parent %llu root %llu owner %llu offset %llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02007011 bytenr, parent, root_objectid, owner_objectid,
7012 owner_offset);
Jeff Mahoney66642832016-06-10 18:19:25 -04007013 btrfs_abort_transaction(trans, ret);
Josef Bacikc4a050b2014-03-14 16:36:53 -04007014 goto out;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01007015 } else {
Jeff Mahoney66642832016-06-10 18:19:25 -04007016 btrfs_abort_transaction(trans, ret);
David Sterba005d6422012-09-18 07:52:32 -06007017 goto out;
Chris Mason7bb86312007-12-11 09:25:06 -05007018 }
Chris Mason5f39d392007-10-15 16:14:19 -04007019
7020 leaf = path->nodes[0];
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007021 item_size = btrfs_item_size_nr(leaf, extent_slot);
David Sterba6d8ff4e2018-06-26 16:20:59 +02007022 if (unlikely(item_size < sizeof(*ei))) {
Nikolay Borisovba3c2b12018-06-26 16:57:36 +03007023 ret = -EINVAL;
7024 btrfs_print_v0_err(info);
7025 btrfs_abort_transaction(trans, ret);
7026 goto out;
7027 }
Chris Mason952fcca2008-02-18 16:33:44 -05007028 ei = btrfs_item_ptr(leaf, extent_slot,
Chris Mason123abc82007-03-14 14:14:43 -04007029 struct btrfs_extent_item);
Josef Bacik3173a182013-03-07 14:22:04 -05007030 if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID &&
7031 key.type == BTRFS_EXTENT_ITEM_KEY) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007032 struct btrfs_tree_block_info *bi;
7033 BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
7034 bi = (struct btrfs_tree_block_info *)(ei + 1);
7035 WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
Chris Mason952fcca2008-02-18 16:33:44 -05007036 }
7037
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007038 refs = btrfs_extent_refs(leaf, ei);
Josef Bacik32b02532013-04-24 16:38:50 -04007039 if (refs < refs_to_drop) {
Jeff Mahoney5d163e02016-09-20 10:05:00 -04007040 btrfs_err(info,
7041 "trying to drop %d refs but we only have %Lu for bytenr %Lu",
7042 refs_to_drop, refs, bytenr);
Josef Bacik32b02532013-04-24 16:38:50 -04007043 ret = -EINVAL;
Jeff Mahoney66642832016-06-10 18:19:25 -04007044 btrfs_abort_transaction(trans, ret);
Josef Bacik32b02532013-04-24 16:38:50 -04007045 goto out;
7046 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007047 refs -= refs_to_drop;
7048
7049 if (refs > 0) {
7050 if (extent_op)
7051 __run_delayed_extent_op(extent_op, leaf, ei);
7052 /*
7053 * In the case of inline back ref, reference count will
7054 * be updated by remove_extent_backref
7055 */
7056 if (iref) {
7057 BUG_ON(!found_extent);
7058 } else {
7059 btrfs_set_extent_refs(leaf, ei, refs);
7060 btrfs_mark_buffer_dirty(leaf);
7061 }
7062 if (found_extent) {
Nikolay Borisov87cc7a82018-06-20 15:49:12 +03007063 ret = remove_extent_backref(trans, path, iref,
7064 refs_to_drop, is_data,
7065 &last_ref);
David Sterba005d6422012-09-18 07:52:32 -06007066 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04007067 btrfs_abort_transaction(trans, ret);
David Sterba005d6422012-09-18 07:52:32 -06007068 goto out;
7069 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007070 }
7071 } else {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007072 if (found_extent) {
7073 BUG_ON(is_data && refs_to_drop !=
Zhaolei9ed0dea2015-08-06 22:16:24 +08007074 extent_data_ref_count(path, iref));
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007075 if (iref) {
7076 BUG_ON(path->slots[0] != extent_slot);
7077 } else {
7078 BUG_ON(path->slots[0] != extent_slot + 1);
7079 path->slots[0] = extent_slot;
7080 num_to_del = 2;
7081 }
Chris Mason78fae272007-03-25 11:35:08 -04007082 }
Chris Masonb9473432009-03-13 11:00:37 -04007083
Josef Bacikfcebe452014-05-13 17:30:47 -07007084 last_ref = 1;
Chris Mason952fcca2008-02-18 16:33:44 -05007085 ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
7086 num_to_del);
David Sterba005d6422012-09-18 07:52:32 -06007087 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04007088 btrfs_abort_transaction(trans, ret);
David Sterba005d6422012-09-18 07:52:32 -06007089 goto out;
7090 }
David Sterbab3b4aa72011-04-21 01:20:15 +02007091 btrfs_release_path(path);
David Woodhouse21af8042008-08-12 14:13:26 +01007092
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007093 if (is_data) {
Jeff Mahoney5b4aace2016-06-21 10:40:19 -04007094 ret = btrfs_del_csums(trans, info, bytenr, num_bytes);
David Sterba005d6422012-09-18 07:52:32 -06007095 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04007096 btrfs_abort_transaction(trans, ret);
David Sterba005d6422012-09-18 07:52:32 -06007097 goto out;
7098 }
Chris Mason459931e2008-12-10 09:10:46 -05007099 }
7100
Nikolay Borisove7355e52018-05-10 15:44:55 +03007101 ret = add_to_free_space_tree(trans, bytenr, num_bytes);
Omar Sandoval1e144fb2015-09-29 20:50:37 -07007102 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04007103 btrfs_abort_transaction(trans, ret);
Omar Sandoval1e144fb2015-09-29 20:50:37 -07007104 goto out;
7105 }
7106
David Sterba6b279402019-03-20 12:10:15 +01007107 ret = update_block_group(trans, bytenr, num_bytes, 0);
David Sterba005d6422012-09-18 07:52:32 -06007108 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04007109 btrfs_abort_transaction(trans, ret);
David Sterba005d6422012-09-18 07:52:32 -06007110 goto out;
7111 }
Chris Masona28ec192007-03-06 20:08:01 -05007112 }
Josef Bacikfcebe452014-05-13 17:30:47 -07007113 btrfs_release_path(path);
7114
Jeff Mahoney79787ea2012-03-12 16:03:00 +01007115out:
Chris Mason5caf2a02007-04-02 11:20:42 -04007116 btrfs_free_path(path);
Chris Masona28ec192007-03-06 20:08:01 -05007117 return ret;
7118}
7119
7120/*
Yan, Zhengf0486c62010-05-16 10:46:25 -04007121 * when we free an block, it is possible (and likely) that we free the last
Chris Mason1887be62009-03-13 10:11:24 -04007122 * delayed ref for that extent as well. This searches the delayed ref tree for
7123 * a given extent, and if there are no other delayed refs to be processed, it
7124 * removes it from the tree.
7125 */
7126static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04007127 u64 bytenr)
Chris Mason1887be62009-03-13 10:11:24 -04007128{
7129 struct btrfs_delayed_ref_head *head;
7130 struct btrfs_delayed_ref_root *delayed_refs;
Yan, Zhengf0486c62010-05-16 10:46:25 -04007131 int ret = 0;
Chris Mason1887be62009-03-13 10:11:24 -04007132
7133 delayed_refs = &trans->transaction->delayed_refs;
7134 spin_lock(&delayed_refs->lock);
Liu Bof72ad18e2017-01-30 12:24:37 -08007135 head = btrfs_find_delayed_ref_head(delayed_refs, bytenr);
Chris Mason1887be62009-03-13 10:11:24 -04007136 if (!head)
Chris Masoncf93da72014-01-29 07:02:40 -08007137 goto out_delayed_unlock;
Chris Mason1887be62009-03-13 10:11:24 -04007138
Josef Bacikd7df2c72014-01-23 09:21:38 -05007139 spin_lock(&head->lock);
Liu Boe3d03962018-08-23 03:51:50 +08007140 if (!RB_EMPTY_ROOT(&head->ref_tree.rb_root))
Chris Mason1887be62009-03-13 10:11:24 -04007141 goto out;
7142
Josef Bacikbedc66172018-12-03 10:20:31 -05007143 if (cleanup_extent_op(head) != NULL)
7144 goto out;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007145
Chris Mason1887be62009-03-13 10:11:24 -04007146 /*
7147 * waiting for the lock here would deadlock. If someone else has it
7148 * locked they are already in the process of dropping it anyway
7149 */
7150 if (!mutex_trylock(&head->mutex))
7151 goto out;
7152
Josef Bacikd7baffd2018-12-03 10:20:29 -05007153 btrfs_delete_ref_head(delayed_refs, head);
Josef Bacikd7df2c72014-01-23 09:21:38 -05007154 head->processing = 0;
Josef Bacikd7baffd2018-12-03 10:20:29 -05007155
Josef Bacikd7df2c72014-01-23 09:21:38 -05007156 spin_unlock(&head->lock);
Chris Mason1887be62009-03-13 10:11:24 -04007157 spin_unlock(&delayed_refs->lock);
7158
Yan, Zhengf0486c62010-05-16 10:46:25 -04007159 BUG_ON(head->extent_op);
7160 if (head->must_insert_reserved)
7161 ret = 1;
7162
Josef Bacik31890da2018-11-21 14:05:41 -05007163 btrfs_cleanup_ref_head_accounting(trans->fs_info, delayed_refs, head);
Yan, Zhengf0486c62010-05-16 10:46:25 -04007164 mutex_unlock(&head->mutex);
Josef Bacikd2788502017-09-29 15:43:57 -04007165 btrfs_put_delayed_ref_head(head);
Yan, Zhengf0486c62010-05-16 10:46:25 -04007166 return ret;
Chris Mason1887be62009-03-13 10:11:24 -04007167out:
Josef Bacikd7df2c72014-01-23 09:21:38 -05007168 spin_unlock(&head->lock);
Chris Masoncf93da72014-01-29 07:02:40 -08007169
7170out_delayed_unlock:
Chris Mason1887be62009-03-13 10:11:24 -04007171 spin_unlock(&delayed_refs->lock);
7172 return 0;
7173}
7174
Yan, Zhengf0486c62010-05-16 10:46:25 -04007175void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
7176 struct btrfs_root *root,
7177 struct extent_buffer *buf,
Jan Schmidt5581a512012-05-16 17:04:52 +02007178 u64 parent, int last_ref)
Yan, Zhengf0486c62010-05-16 10:46:25 -04007179{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04007180 struct btrfs_fs_info *fs_info = root->fs_info;
Qu Wenruoed4f2552019-04-04 14:45:31 +08007181 struct btrfs_ref generic_ref = { 0 };
Josef Bacikb150a4f2013-06-19 15:00:04 -04007182 int pin = 1;
Yan, Zhengf0486c62010-05-16 10:46:25 -04007183 int ret;
7184
Qu Wenruoed4f2552019-04-04 14:45:31 +08007185 btrfs_init_generic_ref(&generic_ref, BTRFS_DROP_DELAYED_REF,
7186 buf->start, buf->len, parent);
7187 btrfs_init_tree_ref(&generic_ref, btrfs_header_level(buf),
7188 root->root_key.objectid);
7189
Yan, Zhengf0486c62010-05-16 10:46:25 -04007190 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
Omar Sandovald7eae342017-06-06 16:45:31 -07007191 int old_ref_mod, new_ref_mod;
7192
Josef Bacikfd708b82017-09-29 15:43:50 -04007193 btrfs_ref_tree_mod(root, buf->start, buf->len, parent,
7194 root->root_key.objectid,
7195 btrfs_header_level(buf), 0,
7196 BTRFS_DROP_DELAYED_REF);
Qu Wenruoed4f2552019-04-04 14:45:31 +08007197 ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, NULL,
Omar Sandovald7eae342017-06-06 16:45:31 -07007198 &old_ref_mod, &new_ref_mod);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01007199 BUG_ON(ret); /* -ENOMEM */
Omar Sandovald7eae342017-06-06 16:45:31 -07007200 pin = old_ref_mod >= 0 && new_ref_mod < 0;
Yan, Zhengf0486c62010-05-16 10:46:25 -04007201 }
7202
Omar Sandoval0a16c7d2017-06-06 16:45:29 -07007203 if (last_ref && btrfs_header_generation(buf) == trans->transid) {
Filipe Manana62198722015-01-06 20:18:45 +00007204 struct btrfs_block_group_cache *cache;
7205
Yan, Zhengf0486c62010-05-16 10:46:25 -04007206 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04007207 ret = check_ref_cleanup(trans, buf->start);
Yan, Zhengf0486c62010-05-16 10:46:25 -04007208 if (!ret)
Josef Bacik37be25b2011-08-05 10:25:38 -04007209 goto out;
Yan, Zhengf0486c62010-05-16 10:46:25 -04007210 }
7211
Omar Sandoval4da8b762017-06-06 16:45:28 -07007212 pin = 0;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04007213 cache = btrfs_lookup_block_group(fs_info, buf->start);
Filipe Manana62198722015-01-06 20:18:45 +00007214
Yan, Zhengf0486c62010-05-16 10:46:25 -04007215 if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
David Sterbafdf08602019-03-20 12:12:32 +01007216 pin_down_extent(cache, buf->start, buf->len, 1);
Filipe Manana62198722015-01-06 20:18:45 +00007217 btrfs_put_block_group(cache);
Josef Bacik37be25b2011-08-05 10:25:38 -04007218 goto out;
Yan, Zhengf0486c62010-05-16 10:46:25 -04007219 }
7220
7221 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));
7222
7223 btrfs_add_free_space(cache, buf->start, buf->len);
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08007224 btrfs_free_reserved_bytes(cache, buf->len, 0);
Filipe Manana62198722015-01-06 20:18:45 +00007225 btrfs_put_block_group(cache);
Jeff Mahoney71ff6432016-09-06 16:00:42 -04007226 trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len);
Yan, Zhengf0486c62010-05-16 10:46:25 -04007227 }
7228out:
Josef Bacikb150a4f2013-06-19 15:00:04 -04007229 if (pin)
Nikolay Borisov29d2b842018-03-30 12:58:47 +03007230 add_pinned_bytes(fs_info, buf->len, true,
Josef Bacikb150a4f2013-06-19 15:00:04 -04007231 root->root_key.objectid);
7232
Omar Sandoval0a16c7d2017-06-06 16:45:29 -07007233 if (last_ref) {
7234 /*
7235 * Deleting the buffer, clear the corrupt flag since it doesn't
7236 * matter anymore.
7237 */
7238 clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
7239 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04007240}
7241
Jeff Mahoney79787ea2012-03-12 16:03:00 +01007242/* Can return -ENOMEM */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04007243int btrfs_free_extent(struct btrfs_trans_handle *trans,
Josef Bacik84f7d8e2017-09-29 15:43:49 -04007244 struct btrfs_root *root,
Arne Jansen66d7e7f2011-09-12 15:26:38 +02007245 u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
Filipe Mananab06c4bf2015-10-23 07:52:54 +01007246 u64 owner, u64 offset)
Chris Mason925baed2008-06-25 16:01:30 -04007247{
Josef Bacik84f7d8e2017-09-29 15:43:49 -04007248 struct btrfs_fs_info *fs_info = root->fs_info;
Qu Wenruoed4f2552019-04-04 14:45:31 +08007249 struct btrfs_ref generic_ref = { 0 };
Omar Sandovald7eae342017-06-06 16:45:31 -07007250 int old_ref_mod, new_ref_mod;
Chris Mason925baed2008-06-25 16:01:30 -04007251 int ret;
7252
Jeff Mahoneyf5ee5c92016-06-21 09:52:41 -04007253 if (btrfs_is_testing(fs_info))
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04007254 return 0;
David Sterbafccb84c2014-09-29 23:53:21 +02007255
Josef Bacikfd708b82017-09-29 15:43:50 -04007256 if (root_objectid != BTRFS_TREE_LOG_OBJECTID)
7257 btrfs_ref_tree_mod(root, bytenr, num_bytes, parent,
7258 root_objectid, owner, offset,
7259 BTRFS_DROP_DELAYED_REF);
7260
Qu Wenruoed4f2552019-04-04 14:45:31 +08007261 btrfs_init_generic_ref(&generic_ref, BTRFS_DROP_DELAYED_REF, bytenr,
7262 num_bytes, parent);
7263 generic_ref.real_root = root->root_key.objectid;
Chris Mason56bec292009-03-13 10:10:06 -04007264 /*
7265 * tree log blocks never actually go into the extent allocation
7266 * tree, just update pinning info and exit early.
Chris Mason56bec292009-03-13 10:10:06 -04007267 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007268 if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
7269 WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
Chris Masonb9473432009-03-13 11:00:37 -04007270 /* unlocks the pinned mutex */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04007271 btrfs_pin_extent(fs_info, bytenr, num_bytes, 1);
Omar Sandovald7eae342017-06-06 16:45:31 -07007272 old_ref_mod = new_ref_mod = 0;
Chris Mason56bec292009-03-13 10:10:06 -04007273 ret = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007274 } else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
Qu Wenruoed4f2552019-04-04 14:45:31 +08007275 btrfs_init_tree_ref(&generic_ref, (int)owner, root_objectid);
7276 ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, NULL,
Omar Sandovald7eae342017-06-06 16:45:31 -07007277 &old_ref_mod, &new_ref_mod);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04007278 } else {
Nikolay Borisov88a979c2018-06-20 15:48:54 +03007279 ret = btrfs_add_delayed_data_ref(trans, bytenr,
Omar Sandoval7be07912017-06-06 16:45:30 -07007280 num_bytes, parent,
7281 root_objectid, owner, offset,
7282 0, BTRFS_DROP_DELAYED_REF,
Omar Sandovald7eae342017-06-06 16:45:31 -07007283 &old_ref_mod, &new_ref_mod);
Chris Mason56bec292009-03-13 10:10:06 -04007284 }
Omar Sandovald7eae342017-06-06 16:45:31 -07007285
Nikolay Borisov29d2b842018-03-30 12:58:47 +03007286 if (ret == 0 && old_ref_mod >= 0 && new_ref_mod < 0) {
7287 bool metadata = owner < BTRFS_FIRST_FREE_OBJECTID;
7288
7289 add_pinned_bytes(fs_info, num_bytes, metadata, root_objectid);
7290 }
Omar Sandovald7eae342017-06-06 16:45:31 -07007291
Chris Mason925baed2008-06-25 16:01:30 -04007292 return ret;
7293}
7294
Chris Masonfec577f2007-02-26 10:40:21 -05007295/*
Josef Bacik817d52f2009-07-13 21:29:25 -04007296 * when we wait for progress in the block group caching, its because
7297 * our allocation attempt failed at least once. So, we must sleep
7298 * and let some progress happen before we try again.
7299 *
7300 * This function will sleep at least once waiting for new free space to
7301 * show up, and then it will check the block group free space numbers
7302 * for our min num_bytes. Another option is to have it go ahead
7303 * and look in the rbtree for a free extent of a given size, but this
7304 * is a good start.
Josef Bacik36cce922013-08-05 11:15:21 -04007305 *
7306 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using
7307 * any of the information in this block group.
Josef Bacik817d52f2009-07-13 21:29:25 -04007308 */
Josef Bacik36cce922013-08-05 11:15:21 -04007309static noinline void
Josef Bacik817d52f2009-07-13 21:29:25 -04007310wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
7311 u64 num_bytes)
7312{
Yan Zheng11833d62009-09-11 16:11:19 -04007313 struct btrfs_caching_control *caching_ctl;
Josef Bacik817d52f2009-07-13 21:29:25 -04007314
Yan Zheng11833d62009-09-11 16:11:19 -04007315 caching_ctl = get_caching_control(cache);
7316 if (!caching_ctl)
Josef Bacik36cce922013-08-05 11:15:21 -04007317 return;
Josef Bacik817d52f2009-07-13 21:29:25 -04007318
Yan Zheng11833d62009-09-11 16:11:19 -04007319 wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
Li Zefan34d52cb2011-03-29 13:46:06 +08007320 (cache->free_space_ctl->free_space >= num_bytes));
Yan Zheng11833d62009-09-11 16:11:19 -04007321
7322 put_caching_control(caching_ctl);
Yan Zheng11833d62009-09-11 16:11:19 -04007323}
7324
7325static noinline int
7326wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
7327{
7328 struct btrfs_caching_control *caching_ctl;
Josef Bacik36cce922013-08-05 11:15:21 -04007329 int ret = 0;
Yan Zheng11833d62009-09-11 16:11:19 -04007330
7331 caching_ctl = get_caching_control(cache);
7332 if (!caching_ctl)
Josef Bacik36cce922013-08-05 11:15:21 -04007333 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
Yan Zheng11833d62009-09-11 16:11:19 -04007334
7335 wait_event(caching_ctl->wait, block_group_cache_done(cache));
Josef Bacik36cce922013-08-05 11:15:21 -04007336 if (cache->cached == BTRFS_CACHE_ERROR)
7337 ret = -EIO;
Yan Zheng11833d62009-09-11 16:11:19 -04007338 put_caching_control(caching_ctl);
Josef Bacik36cce922013-08-05 11:15:21 -04007339 return ret;
Josef Bacik817d52f2009-07-13 21:29:25 -04007340}
7341
7342enum btrfs_loop_type {
Josef Bacik285ff5a2012-01-13 15:27:45 -05007343 LOOP_CACHING_NOWAIT = 0,
7344 LOOP_CACHING_WAIT = 1,
7345 LOOP_ALLOC_CHUNK = 2,
7346 LOOP_NO_EMPTY_SIZE = 3,
Josef Bacik817d52f2009-07-13 21:29:25 -04007347};
7348
Miao Xiee570fd22014-06-19 10:42:50 +08007349static inline void
7350btrfs_lock_block_group(struct btrfs_block_group_cache *cache,
7351 int delalloc)
7352{
7353 if (delalloc)
7354 down_read(&cache->data_rwsem);
7355}
7356
7357static inline void
7358btrfs_grab_block_group(struct btrfs_block_group_cache *cache,
7359 int delalloc)
7360{
7361 btrfs_get_block_group(cache);
7362 if (delalloc)
7363 down_read(&cache->data_rwsem);
7364}
7365
7366static struct btrfs_block_group_cache *
7367btrfs_lock_cluster(struct btrfs_block_group_cache *block_group,
7368 struct btrfs_free_cluster *cluster,
7369 int delalloc)
7370{
Sudip Mukherjee89771cc2016-02-16 13:32:47 +05307371 struct btrfs_block_group_cache *used_bg = NULL;
Geert Uytterhoeven6719afd2014-06-22 14:30:09 +02007372
Miao Xiee570fd22014-06-19 10:42:50 +08007373 spin_lock(&cluster->refill_lock);
Geert Uytterhoeven6719afd2014-06-22 14:30:09 +02007374 while (1) {
7375 used_bg = cluster->block_group;
7376 if (!used_bg)
7377 return NULL;
7378
7379 if (used_bg == block_group)
7380 return used_bg;
7381
7382 btrfs_get_block_group(used_bg);
7383
7384 if (!delalloc)
7385 return used_bg;
7386
7387 if (down_read_trylock(&used_bg->data_rwsem))
7388 return used_bg;
7389
7390 spin_unlock(&cluster->refill_lock);
7391
Liu Boe321f8a2016-11-30 16:11:04 -08007392 /* We should only have one-level nested. */
7393 down_read_nested(&used_bg->data_rwsem, SINGLE_DEPTH_NESTING);
Geert Uytterhoeven6719afd2014-06-22 14:30:09 +02007394
7395 spin_lock(&cluster->refill_lock);
Miao Xiee570fd22014-06-19 10:42:50 +08007396 if (used_bg == cluster->block_group)
7397 return used_bg;
7398
7399 up_read(&used_bg->data_rwsem);
7400 btrfs_put_block_group(used_bg);
7401 }
Miao Xiee570fd22014-06-19 10:42:50 +08007402}
7403
7404static inline void
7405btrfs_release_block_group(struct btrfs_block_group_cache *cache,
7406 int delalloc)
7407{
7408 if (delalloc)
7409 up_read(&cache->data_rwsem);
7410 btrfs_put_block_group(cache);
7411}
7412
Josef Bacik817d52f2009-07-13 21:29:25 -04007413/*
Qu Wenruob4bd7452018-11-02 09:39:47 +08007414 * Structure used internally for find_free_extent() function. Wraps needed
7415 * parameters.
7416 */
7417struct find_free_extent_ctl {
7418 /* Basic allocation info */
7419 u64 ram_bytes;
7420 u64 num_bytes;
7421 u64 empty_size;
7422 u64 flags;
7423 int delalloc;
7424
7425 /* Where to start the search inside the bg */
7426 u64 search_start;
7427
7428 /* For clustered allocation */
7429 u64 empty_cluster;
7430
7431 bool have_caching_bg;
7432 bool orig_have_caching_bg;
7433
7434 /* RAID index, converted from flags */
7435 int index;
7436
Qu Wenruoe72d79d2018-11-02 09:39:50 +08007437 /*
7438 * Current loop number, check find_free_extent_update_loop() for details
7439 */
Qu Wenruob4bd7452018-11-02 09:39:47 +08007440 int loop;
7441
7442 /*
7443 * Whether we're refilling a cluster, if true we need to re-search
7444 * current block group but don't try to refill the cluster again.
7445 */
7446 bool retry_clustered;
7447
7448 /*
7449 * Whether we're updating free space cache, if true we need to re-search
7450 * current block group but don't try updating free space cache again.
7451 */
7452 bool retry_unclustered;
7453
7454 /* If current block group is cached */
7455 int cached;
7456
7457 /* Max contiguous hole found */
7458 u64 max_extent_size;
7459
7460 /* Total free space from free space cache, not always contiguous */
7461 u64 total_free_space;
7462
7463 /* Found result */
7464 u64 found_offset;
7465};
7466
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007467
7468/*
7469 * Helper function for find_free_extent().
7470 *
7471 * Return -ENOENT to inform caller that we need fallback to unclustered mode.
7472 * Return -EAGAIN to inform caller that we need to re-search this block group
7473 * Return >0 to inform caller that we find nothing
7474 * Return 0 means we have found a location and set ffe_ctl->found_offset.
7475 */
7476static int find_free_extent_clustered(struct btrfs_block_group_cache *bg,
7477 struct btrfs_free_cluster *last_ptr,
7478 struct find_free_extent_ctl *ffe_ctl,
7479 struct btrfs_block_group_cache **cluster_bg_ret)
7480{
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007481 struct btrfs_block_group_cache *cluster_bg;
7482 u64 aligned_cluster;
7483 u64 offset;
7484 int ret;
7485
7486 cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc);
7487 if (!cluster_bg)
7488 goto refill_cluster;
7489 if (cluster_bg != bg && (cluster_bg->ro ||
7490 !block_group_bits(cluster_bg, ffe_ctl->flags)))
7491 goto release_cluster;
7492
7493 offset = btrfs_alloc_from_cluster(cluster_bg, last_ptr,
7494 ffe_ctl->num_bytes, cluster_bg->key.objectid,
7495 &ffe_ctl->max_extent_size);
7496 if (offset) {
7497 /* We have a block, we're done */
7498 spin_unlock(&last_ptr->refill_lock);
7499 trace_btrfs_reserve_extent_cluster(cluster_bg,
7500 ffe_ctl->search_start, ffe_ctl->num_bytes);
7501 *cluster_bg_ret = cluster_bg;
7502 ffe_ctl->found_offset = offset;
7503 return 0;
7504 }
7505 WARN_ON(last_ptr->block_group != cluster_bg);
7506
7507release_cluster:
7508 /*
7509 * If we are on LOOP_NO_EMPTY_SIZE, we can't set up a new clusters, so
7510 * lets just skip it and let the allocator find whatever block it can
7511 * find. If we reach this point, we will have tried the cluster
7512 * allocator plenty of times and not have found anything, so we are
7513 * likely way too fragmented for the clustering stuff to find anything.
7514 *
7515 * However, if the cluster is taken from the current block group,
7516 * release the cluster first, so that we stand a better chance of
7517 * succeeding in the unclustered allocation.
7518 */
7519 if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) {
7520 spin_unlock(&last_ptr->refill_lock);
7521 btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);
7522 return -ENOENT;
7523 }
7524
7525 /* This cluster didn't work out, free it and start over */
7526 btrfs_return_cluster_to_free_space(NULL, last_ptr);
7527
7528 if (cluster_bg != bg)
7529 btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);
7530
7531refill_cluster:
7532 if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) {
7533 spin_unlock(&last_ptr->refill_lock);
7534 return -ENOENT;
7535 }
7536
7537 aligned_cluster = max_t(u64,
7538 ffe_ctl->empty_cluster + ffe_ctl->empty_size,
7539 bg->full_stripe_len);
David Sterba2ceeae22019-03-20 13:53:49 +01007540 ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start,
7541 ffe_ctl->num_bytes, aligned_cluster);
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007542 if (ret == 0) {
7543 /* Now pull our allocation out of this cluster */
7544 offset = btrfs_alloc_from_cluster(bg, last_ptr,
7545 ffe_ctl->num_bytes, ffe_ctl->search_start,
7546 &ffe_ctl->max_extent_size);
7547 if (offset) {
7548 /* We found one, proceed */
7549 spin_unlock(&last_ptr->refill_lock);
7550 trace_btrfs_reserve_extent_cluster(bg,
7551 ffe_ctl->search_start,
7552 ffe_ctl->num_bytes);
7553 ffe_ctl->found_offset = offset;
7554 return 0;
7555 }
7556 } else if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT &&
7557 !ffe_ctl->retry_clustered) {
7558 spin_unlock(&last_ptr->refill_lock);
7559
7560 ffe_ctl->retry_clustered = true;
7561 wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +
7562 ffe_ctl->empty_cluster + ffe_ctl->empty_size);
7563 return -EAGAIN;
7564 }
7565 /*
7566 * At this point we either didn't find a cluster or we weren't able to
7567 * allocate a block from our cluster. Free the cluster we've been
7568 * trying to use, and go to the next block group.
7569 */
7570 btrfs_return_cluster_to_free_space(NULL, last_ptr);
7571 spin_unlock(&last_ptr->refill_lock);
7572 return 1;
7573}
7574
Qu Wenruob4bd7452018-11-02 09:39:47 +08007575/*
Qu Wenruoe1a41842018-11-02 09:39:49 +08007576 * Return >0 to inform caller that we find nothing
7577 * Return 0 when we found an free extent and set ffe_ctrl->found_offset
7578 * Return -EAGAIN to inform caller that we need to re-search this block group
7579 */
7580static int find_free_extent_unclustered(struct btrfs_block_group_cache *bg,
7581 struct btrfs_free_cluster *last_ptr,
7582 struct find_free_extent_ctl *ffe_ctl)
7583{
7584 u64 offset;
7585
7586 /*
7587 * We are doing an unclustered allocation, set the fragmented flag so
7588 * we don't bother trying to setup a cluster again until we get more
7589 * space.
7590 */
7591 if (unlikely(last_ptr)) {
7592 spin_lock(&last_ptr->lock);
7593 last_ptr->fragmented = 1;
7594 spin_unlock(&last_ptr->lock);
7595 }
7596 if (ffe_ctl->cached) {
7597 struct btrfs_free_space_ctl *free_space_ctl;
7598
7599 free_space_ctl = bg->free_space_ctl;
7600 spin_lock(&free_space_ctl->tree_lock);
7601 if (free_space_ctl->free_space <
7602 ffe_ctl->num_bytes + ffe_ctl->empty_cluster +
7603 ffe_ctl->empty_size) {
7604 ffe_ctl->total_free_space = max_t(u64,
7605 ffe_ctl->total_free_space,
7606 free_space_ctl->free_space);
7607 spin_unlock(&free_space_ctl->tree_lock);
7608 return 1;
7609 }
7610 spin_unlock(&free_space_ctl->tree_lock);
7611 }
7612
7613 offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start,
7614 ffe_ctl->num_bytes, ffe_ctl->empty_size,
7615 &ffe_ctl->max_extent_size);
7616
7617 /*
7618 * If we didn't find a chunk, and we haven't failed on this block group
7619 * before, and this block group is in the middle of caching and we are
7620 * ok with waiting, then go ahead and wait for progress to be made, and
7621 * set @retry_unclustered to true.
7622 *
7623 * If @retry_unclustered is true then we've already waited on this
7624 * block group once and should move on to the next block group.
7625 */
7626 if (!offset && !ffe_ctl->retry_unclustered && !ffe_ctl->cached &&
7627 ffe_ctl->loop > LOOP_CACHING_NOWAIT) {
7628 wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +
7629 ffe_ctl->empty_size);
7630 ffe_ctl->retry_unclustered = true;
7631 return -EAGAIN;
7632 } else if (!offset) {
7633 return 1;
7634 }
7635 ffe_ctl->found_offset = offset;
7636 return 0;
7637}
7638
7639/*
Qu Wenruoe72d79d2018-11-02 09:39:50 +08007640 * Return >0 means caller needs to re-search for free extent
7641 * Return 0 means we have the needed free extent.
7642 * Return <0 means we failed to locate any free extent.
7643 */
7644static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info,
7645 struct btrfs_free_cluster *last_ptr,
7646 struct btrfs_key *ins,
7647 struct find_free_extent_ctl *ffe_ctl,
7648 int full_search, bool use_cluster)
7649{
7650 struct btrfs_root *root = fs_info->extent_root;
7651 int ret;
7652
7653 if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) &&
7654 ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg)
7655 ffe_ctl->orig_have_caching_bg = true;
7656
7657 if (!ins->objectid && ffe_ctl->loop >= LOOP_CACHING_WAIT &&
7658 ffe_ctl->have_caching_bg)
7659 return 1;
7660
7661 if (!ins->objectid && ++(ffe_ctl->index) < BTRFS_NR_RAID_TYPES)
7662 return 1;
7663
7664 if (ins->objectid) {
7665 if (!use_cluster && last_ptr) {
7666 spin_lock(&last_ptr->lock);
7667 last_ptr->window_start = ins->objectid;
7668 spin_unlock(&last_ptr->lock);
7669 }
7670 return 0;
7671 }
7672
7673 /*
7674 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
7675 * caching kthreads as we move along
7676 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
7677 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
7678 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
7679 * again
7680 */
7681 if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) {
7682 ffe_ctl->index = 0;
7683 if (ffe_ctl->loop == LOOP_CACHING_NOWAIT) {
7684 /*
7685 * We want to skip the LOOP_CACHING_WAIT step if we
7686 * don't have any uncached bgs and we've already done a
7687 * full search through.
7688 */
7689 if (ffe_ctl->orig_have_caching_bg || !full_search)
7690 ffe_ctl->loop = LOOP_CACHING_WAIT;
7691 else
7692 ffe_ctl->loop = LOOP_ALLOC_CHUNK;
7693 } else {
7694 ffe_ctl->loop++;
7695 }
7696
7697 if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) {
7698 struct btrfs_trans_handle *trans;
7699 int exist = 0;
7700
7701 trans = current->journal_info;
7702 if (trans)
7703 exist = 1;
7704 else
7705 trans = btrfs_join_transaction(root);
7706
7707 if (IS_ERR(trans)) {
7708 ret = PTR_ERR(trans);
7709 return ret;
7710 }
7711
7712 ret = do_chunk_alloc(trans, ffe_ctl->flags,
7713 CHUNK_ALLOC_FORCE);
7714
7715 /*
7716 * If we can't allocate a new chunk we've already looped
7717 * through at least once, move on to the NO_EMPTY_SIZE
7718 * case.
7719 */
7720 if (ret == -ENOSPC)
7721 ffe_ctl->loop = LOOP_NO_EMPTY_SIZE;
7722
7723 /* Do not bail out on ENOSPC since we can do more. */
7724 if (ret < 0 && ret != -ENOSPC)
7725 btrfs_abort_transaction(trans, ret);
7726 else
7727 ret = 0;
7728 if (!exist)
7729 btrfs_end_transaction(trans);
7730 if (ret)
7731 return ret;
7732 }
7733
7734 if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) {
7735 /*
7736 * Don't loop again if we already have no empty_size and
7737 * no empty_cluster.
7738 */
7739 if (ffe_ctl->empty_size == 0 &&
7740 ffe_ctl->empty_cluster == 0)
7741 return -ENOSPC;
7742 ffe_ctl->empty_size = 0;
7743 ffe_ctl->empty_cluster = 0;
7744 }
7745 return 1;
7746 }
7747 return -ENOSPC;
7748}
7749
7750/*
Chris Masonfec577f2007-02-26 10:40:21 -05007751 * walks the btree of allocated extents and find a hole of a given size.
7752 * The key ins is changed to record the hole:
Miao Xiea4820392013-09-09 13:19:42 +08007753 * ins->objectid == start position
Chris Mason62e27492007-03-15 12:56:47 -04007754 * ins->flags = BTRFS_EXTENT_ITEM_KEY
Miao Xiea4820392013-09-09 13:19:42 +08007755 * ins->offset == the size of the hole.
Chris Masonfec577f2007-02-26 10:40:21 -05007756 * Any available blocks before search_start are skipped.
Miao Xiea4820392013-09-09 13:19:42 +08007757 *
7758 * If there is no suitable free space, we will record the max size of
7759 * the free space extent currently.
Qu Wenruoe72d79d2018-11-02 09:39:50 +08007760 *
7761 * The overall logic and call chain:
7762 *
7763 * find_free_extent()
7764 * |- Iterate through all block groups
7765 * | |- Get a valid block group
7766 * | |- Try to do clustered allocation in that block group
7767 * | |- Try to do unclustered allocation in that block group
7768 * | |- Check if the result is valid
7769 * | | |- If valid, then exit
7770 * | |- Jump to next block group
7771 * |
7772 * |- Push harder to find free extents
7773 * |- If not found, re-iterate all block groups
Chris Masonfec577f2007-02-26 10:40:21 -05007774 */
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05007775static noinline int find_free_extent(struct btrfs_fs_info *fs_info,
Wang Xiaoguang18513092016-07-25 15:51:40 +08007776 u64 ram_bytes, u64 num_bytes, u64 empty_size,
7777 u64 hint_byte, struct btrfs_key *ins,
7778 u64 flags, int delalloc)
Chris Masonfec577f2007-02-26 10:40:21 -05007779{
Josef Bacik80eb2342008-10-29 14:49:05 -04007780 int ret = 0;
Chris Masonfa9c0d792009-04-03 09:47:43 -04007781 struct btrfs_free_cluster *last_ptr = NULL;
Josef Bacik80eb2342008-10-29 14:49:05 -04007782 struct btrfs_block_group_cache *block_group = NULL;
Qu Wenruob4bd7452018-11-02 09:39:47 +08007783 struct find_free_extent_ctl ffe_ctl = {0};
Josef Bacik80eb2342008-10-29 14:49:05 -04007784 struct btrfs_space_info *space_info;
Josef Bacik67377732010-09-16 16:19:09 -04007785 bool use_cluster = true;
Josef Bacika5e681d2015-10-01 14:54:10 -04007786 bool full_search = false;
Chris Masonfec577f2007-02-26 10:40:21 -05007787
Jeff Mahoney0b246af2016-06-22 18:54:23 -04007788 WARN_ON(num_bytes < fs_info->sectorsize);
Qu Wenruob4bd7452018-11-02 09:39:47 +08007789
7790 ffe_ctl.ram_bytes = ram_bytes;
7791 ffe_ctl.num_bytes = num_bytes;
7792 ffe_ctl.empty_size = empty_size;
7793 ffe_ctl.flags = flags;
7794 ffe_ctl.search_start = 0;
7795 ffe_ctl.retry_clustered = false;
7796 ffe_ctl.retry_unclustered = false;
7797 ffe_ctl.delalloc = delalloc;
7798 ffe_ctl.index = btrfs_bg_flags_to_raid_index(flags);
7799 ffe_ctl.have_caching_bg = false;
7800 ffe_ctl.orig_have_caching_bg = false;
7801 ffe_ctl.found_offset = 0;
7802
David Sterba962a2982014-06-04 18:41:45 +02007803 ins->type = BTRFS_EXTENT_ITEM_KEY;
Josef Bacik80eb2342008-10-29 14:49:05 -04007804 ins->objectid = 0;
7805 ins->offset = 0;
Chris Masonb1a4d962007-04-04 15:27:52 -04007806
Jeff Mahoney71ff6432016-09-06 16:00:42 -04007807 trace_find_free_extent(fs_info, num_bytes, empty_size, flags);
Josef Bacik3f7de032011-11-10 08:29:20 -05007808
Jeff Mahoney0b246af2016-06-22 18:54:23 -04007809 space_info = __find_space_info(fs_info, flags);
Josef Bacik1b1d1f62010-03-19 20:49:55 +00007810 if (!space_info) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04007811 btrfs_err(fs_info, "No space info for %llu", flags);
Josef Bacik1b1d1f62010-03-19 20:49:55 +00007812 return -ENOSPC;
7813 }
Josef Bacik2552d172009-04-03 10:14:19 -04007814
Josef Bacik67377732010-09-16 16:19:09 -04007815 /*
Josef Bacik4f4db212015-09-29 11:40:47 -04007816 * If our free space is heavily fragmented we may not be able to make
7817 * big contiguous allocations, so instead of doing the expensive search
7818 * for free space, simply return ENOSPC with our max_extent_size so we
7819 * can go ahead and search for a more manageable chunk.
7820 *
7821 * If our max_extent_size is large enough for our allocation simply
7822 * disable clustering since we will likely not be able to find enough
7823 * space to create a cluster and induce latency trying.
Josef Bacik67377732010-09-16 16:19:09 -04007824 */
Josef Bacik4f4db212015-09-29 11:40:47 -04007825 if (unlikely(space_info->max_extent_size)) {
7826 spin_lock(&space_info->lock);
7827 if (space_info->max_extent_size &&
7828 num_bytes > space_info->max_extent_size) {
7829 ins->offset = space_info->max_extent_size;
7830 spin_unlock(&space_info->lock);
7831 return -ENOSPC;
7832 } else if (space_info->max_extent_size) {
7833 use_cluster = false;
7834 }
7835 spin_unlock(&space_info->lock);
Chris Mason239b14b2008-03-24 15:02:07 -04007836 }
7837
Qu Wenruob4bd7452018-11-02 09:39:47 +08007838 last_ptr = fetch_cluster_info(fs_info, space_info,
7839 &ffe_ctl.empty_cluster);
Chris Mason239b14b2008-03-24 15:02:07 -04007840 if (last_ptr) {
Chris Masonfa9c0d792009-04-03 09:47:43 -04007841 spin_lock(&last_ptr->lock);
7842 if (last_ptr->block_group)
7843 hint_byte = last_ptr->window_start;
Josef Bacikc759c4e2015-10-02 15:25:10 -04007844 if (last_ptr->fragmented) {
7845 /*
7846 * We still set window_start so we can keep track of the
7847 * last place we found an allocation to try and save
7848 * some time.
7849 */
7850 hint_byte = last_ptr->window_start;
7851 use_cluster = false;
7852 }
Chris Masonfa9c0d792009-04-03 09:47:43 -04007853 spin_unlock(&last_ptr->lock);
Chris Mason239b14b2008-03-24 15:02:07 -04007854 }
Chris Masonfa9c0d792009-04-03 09:47:43 -04007855
Qu Wenruob4bd7452018-11-02 09:39:47 +08007856 ffe_ctl.search_start = max(ffe_ctl.search_start,
7857 first_logical_byte(fs_info, 0));
7858 ffe_ctl.search_start = max(ffe_ctl.search_start, hint_byte);
7859 if (ffe_ctl.search_start == hint_byte) {
7860 block_group = btrfs_lookup_block_group(fs_info,
7861 ffe_ctl.search_start);
Josef Bacik817d52f2009-07-13 21:29:25 -04007862 /*
7863 * we don't want to use the block group if it doesn't match our
7864 * allocation bits, or if its not cached.
Josef Bacikccf0e722009-11-10 21:23:48 -05007865 *
7866 * However if we are re-searching with an ideal block group
7867 * picked out then we don't care that the block group is cached.
Josef Bacik817d52f2009-07-13 21:29:25 -04007868 */
David Sterbab6919a52013-04-29 13:39:40 +00007869 if (block_group && block_group_bits(block_group, flags) &&
Josef Bacik285ff5a2012-01-13 15:27:45 -05007870 block_group->cached != BTRFS_CACHE_NO) {
Josef Bacik2552d172009-04-03 10:14:19 -04007871 down_read(&space_info->groups_sem);
Chris Mason44fb5512009-06-04 15:34:51 -04007872 if (list_empty(&block_group->list) ||
7873 block_group->ro) {
7874 /*
7875 * someone is removing this block group,
7876 * we can't jump into the have_block_group
7877 * target because our list pointers are not
7878 * valid
7879 */
7880 btrfs_put_block_group(block_group);
7881 up_read(&space_info->groups_sem);
Josef Bacikccf0e722009-11-10 21:23:48 -05007882 } else {
Qu Wenruob4bd7452018-11-02 09:39:47 +08007883 ffe_ctl.index = btrfs_bg_flags_to_raid_index(
Qu Wenruo3e72ee82018-01-30 18:20:45 +08007884 block_group->flags);
Miao Xiee570fd22014-06-19 10:42:50 +08007885 btrfs_lock_block_group(block_group, delalloc);
Chris Mason44fb5512009-06-04 15:34:51 -04007886 goto have_block_group;
Josef Bacikccf0e722009-11-10 21:23:48 -05007887 }
Josef Bacik2552d172009-04-03 10:14:19 -04007888 } else if (block_group) {
Chris Masonfa9c0d792009-04-03 09:47:43 -04007889 btrfs_put_block_group(block_group);
Josef Bacik2552d172009-04-03 10:14:19 -04007890 }
Chris Mason42e70e72008-11-07 18:17:11 -05007891 }
Josef Bacik2552d172009-04-03 10:14:19 -04007892search:
Qu Wenruob4bd7452018-11-02 09:39:47 +08007893 ffe_ctl.have_caching_bg = false;
7894 if (ffe_ctl.index == btrfs_bg_flags_to_raid_index(flags) ||
7895 ffe_ctl.index == 0)
Josef Bacika5e681d2015-10-01 14:54:10 -04007896 full_search = true;
Josef Bacik80eb2342008-10-29 14:49:05 -04007897 down_read(&space_info->groups_sem);
Qu Wenruob4bd7452018-11-02 09:39:47 +08007898 list_for_each_entry(block_group,
7899 &space_info->block_groups[ffe_ctl.index], list) {
Jeff Mahoney14443932017-07-19 23:25:51 -04007900 /* If the block group is read-only, we can skip it entirely. */
7901 if (unlikely(block_group->ro))
7902 continue;
7903
Miao Xiee570fd22014-06-19 10:42:50 +08007904 btrfs_grab_block_group(block_group, delalloc);
Qu Wenruob4bd7452018-11-02 09:39:47 +08007905 ffe_ctl.search_start = block_group->key.objectid;
Chris Mason42e70e72008-11-07 18:17:11 -05007906
Chris Mason83a50de2010-12-13 15:06:46 -05007907 /*
7908 * this can happen if we end up cycling through all the
7909 * raid types, but we want to make sure we only allocate
7910 * for the proper type.
7911 */
David Sterbab6919a52013-04-29 13:39:40 +00007912 if (!block_group_bits(block_group, flags)) {
Bart Van Asschebece2e82018-06-20 10:03:31 -07007913 u64 extra = BTRFS_BLOCK_GROUP_DUP |
Chris Mason83a50de2010-12-13 15:06:46 -05007914 BTRFS_BLOCK_GROUP_RAID1 |
David Woodhouse53b381b2013-01-29 18:40:14 -05007915 BTRFS_BLOCK_GROUP_RAID5 |
7916 BTRFS_BLOCK_GROUP_RAID6 |
Chris Mason83a50de2010-12-13 15:06:46 -05007917 BTRFS_BLOCK_GROUP_RAID10;
7918
7919 /*
7920 * if they asked for extra copies and this block group
7921 * doesn't provide them, bail. This does allow us to
7922 * fill raid0 from raid1.
7923 */
David Sterbab6919a52013-04-29 13:39:40 +00007924 if ((flags & extra) && !(block_group->flags & extra))
Chris Mason83a50de2010-12-13 15:06:46 -05007925 goto loop;
7926 }
7927
Josef Bacik2552d172009-04-03 10:14:19 -04007928have_block_group:
Qu Wenruob4bd7452018-11-02 09:39:47 +08007929 ffe_ctl.cached = block_group_cache_done(block_group);
7930 if (unlikely(!ffe_ctl.cached)) {
7931 ffe_ctl.have_caching_bg = true;
Liu Bof6373bf2012-12-27 09:01:18 +00007932 ret = cache_block_group(block_group, 0);
Chris Mason1d4284b2012-03-28 20:31:37 -04007933 BUG_ON(ret < 0);
7934 ret = 0;
Josef Bacikea6a4782008-11-20 12:16:16 -05007935 }
7936
Josef Bacik36cce922013-08-05 11:15:21 -04007937 if (unlikely(block_group->cached == BTRFS_CACHE_ERROR))
7938 goto loop;
Josef Bacik0f9dd462008-09-23 13:14:11 -04007939
Josef Bacik0a243252009-09-11 16:11:20 -04007940 /*
Alexandre Oliva062c05c2011-12-07 19:50:42 -05007941 * Ok we want to try and use the cluster allocator, so
7942 * lets look there
Josef Bacik0a243252009-09-11 16:11:20 -04007943 */
Josef Bacikc759c4e2015-10-02 15:25:10 -04007944 if (last_ptr && use_cluster) {
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007945 struct btrfs_block_group_cache *cluster_bg = NULL;
Chris Mason44fb5512009-06-04 15:34:51 -04007946
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007947 ret = find_free_extent_clustered(block_group, last_ptr,
7948 &ffe_ctl, &cluster_bg);
Alexandre Oliva274bd4f2011-12-07 20:08:40 -05007949
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007950 if (ret == 0) {
7951 if (cluster_bg && cluster_bg != block_group) {
Miao Xiee570fd22014-06-19 10:42:50 +08007952 btrfs_release_block_group(block_group,
7953 delalloc);
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007954 block_group = cluster_bg;
Miao Xie215a63d2014-01-15 20:00:56 +08007955 }
Chris Masonfa9c0d792009-04-03 09:47:43 -04007956 goto checks;
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007957 } else if (ret == -EAGAIN) {
Josef Bacik817d52f2009-07-13 21:29:25 -04007958 goto have_block_group;
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007959 } else if (ret > 0) {
7960 goto loop;
Chris Masonfa9c0d792009-04-03 09:47:43 -04007961 }
Qu Wenruod06e3bb2018-11-02 09:39:48 +08007962 /* ret == -ENOENT case falls through */
Chris Masonfa9c0d792009-04-03 09:47:43 -04007963 }
7964
Qu Wenruoe1a41842018-11-02 09:39:49 +08007965 ret = find_free_extent_unclustered(block_group, last_ptr,
7966 &ffe_ctl);
7967 if (ret == -EAGAIN)
Josef Bacik817d52f2009-07-13 21:29:25 -04007968 goto have_block_group;
Qu Wenruoe1a41842018-11-02 09:39:49 +08007969 else if (ret > 0)
Josef Bacik1cdda9b2009-10-06 10:04:28 -04007970 goto loop;
Qu Wenruoe1a41842018-11-02 09:39:49 +08007971 /* ret == 0 case falls through */
Chris Masonfa9c0d792009-04-03 09:47:43 -04007972checks:
Qu Wenruob4bd7452018-11-02 09:39:47 +08007973 ffe_ctl.search_start = round_up(ffe_ctl.found_offset,
7974 fs_info->stripesize);
Chris Masone37c9e62007-05-09 20:13:14 -04007975
Josef Bacik2552d172009-04-03 10:14:19 -04007976 /* move on to the next group */
Qu Wenruob4bd7452018-11-02 09:39:47 +08007977 if (ffe_ctl.search_start + num_bytes >
Miao Xie215a63d2014-01-15 20:00:56 +08007978 block_group->key.objectid + block_group->key.offset) {
Qu Wenruob4bd7452018-11-02 09:39:47 +08007979 btrfs_add_free_space(block_group, ffe_ctl.found_offset,
7980 num_bytes);
Josef Bacik2552d172009-04-03 10:14:19 -04007981 goto loop;
Josef Bacik6226cb02009-04-03 10:14:18 -04007982 }
Josef Bacik80eb2342008-10-29 14:49:05 -04007983
Qu Wenruob4bd7452018-11-02 09:39:47 +08007984 if (ffe_ctl.found_offset < ffe_ctl.search_start)
7985 btrfs_add_free_space(block_group, ffe_ctl.found_offset,
7986 ffe_ctl.search_start - ffe_ctl.found_offset);
Josef Bacik6226cb02009-04-03 10:14:18 -04007987
Wang Xiaoguang18513092016-07-25 15:51:40 +08007988 ret = btrfs_add_reserved_bytes(block_group, ram_bytes,
7989 num_bytes, delalloc);
Yan, Zhengf0486c62010-05-16 10:46:25 -04007990 if (ret == -EAGAIN) {
Qu Wenruob4bd7452018-11-02 09:39:47 +08007991 btrfs_add_free_space(block_group, ffe_ctl.found_offset,
7992 num_bytes);
Yan, Zhengf0486c62010-05-16 10:46:25 -04007993 goto loop;
7994 }
Filipe Manana9cfa3e32016-04-26 15:39:32 +01007995 btrfs_inc_block_group_reservations(block_group);
Yan Zheng11833d62009-09-11 16:11:19 -04007996
Josef Bacik2552d172009-04-03 10:14:19 -04007997 /* we are all good, lets return */
Qu Wenruob4bd7452018-11-02 09:39:47 +08007998 ins->objectid = ffe_ctl.search_start;
Yan, Zhengf0486c62010-05-16 10:46:25 -04007999 ins->offset = num_bytes;
8000
Qu Wenruob4bd7452018-11-02 09:39:47 +08008001 trace_btrfs_reserve_extent(block_group, ffe_ctl.search_start,
8002 num_bytes);
Miao Xiee570fd22014-06-19 10:42:50 +08008003 btrfs_release_block_group(block_group, delalloc);
Josef Bacik2552d172009-04-03 10:14:19 -04008004 break;
8005loop:
Qu Wenruob4bd7452018-11-02 09:39:47 +08008006 ffe_ctl.retry_clustered = false;
8007 ffe_ctl.retry_unclustered = false;
Qu Wenruo3e72ee82018-01-30 18:20:45 +08008008 BUG_ON(btrfs_bg_flags_to_raid_index(block_group->flags) !=
Qu Wenruob4bd7452018-11-02 09:39:47 +08008009 ffe_ctl.index);
Miao Xiee570fd22014-06-19 10:42:50 +08008010 btrfs_release_block_group(block_group, delalloc);
Jeff Mahoney14443932017-07-19 23:25:51 -04008011 cond_resched();
Josef Bacik2552d172009-04-03 10:14:19 -04008012 }
8013 up_read(&space_info->groups_sem);
Chris Masonf5a31e12008-11-10 11:47:09 -05008014
Qu Wenruoe72d79d2018-11-02 09:39:50 +08008015 ret = find_free_extent_update_loop(fs_info, last_ptr, ins, &ffe_ctl,
8016 full_search, use_cluster);
8017 if (ret > 0)
Miao Xie60d2adb2011-09-09 17:34:35 +08008018 goto search;
8019
Josef Bacik4f4db212015-09-29 11:40:47 -04008020 if (ret == -ENOSPC) {
Qu Wenruob4bd7452018-11-02 09:39:47 +08008021 /*
8022 * Use ffe_ctl->total_free_space as fallback if we can't find
8023 * any contiguous hole.
8024 */
8025 if (!ffe_ctl.max_extent_size)
8026 ffe_ctl.max_extent_size = ffe_ctl.total_free_space;
Josef Bacik4f4db212015-09-29 11:40:47 -04008027 spin_lock(&space_info->lock);
Qu Wenruob4bd7452018-11-02 09:39:47 +08008028 space_info->max_extent_size = ffe_ctl.max_extent_size;
Josef Bacik4f4db212015-09-29 11:40:47 -04008029 spin_unlock(&space_info->lock);
Qu Wenruob4bd7452018-11-02 09:39:47 +08008030 ins->offset = ffe_ctl.max_extent_size;
Josef Bacik4f4db212015-09-29 11:40:47 -04008031 }
Chris Mason0f70abe2007-02-28 16:46:22 -05008032 return ret;
Chris Masonfec577f2007-02-26 10:40:21 -05008033}
Chris Masonec44a352008-04-28 15:29:52 -04008034
Josef Bacikb78e5612018-11-21 14:03:07 -05008035#define DUMP_BLOCK_RSV(fs_info, rsv_name) \
8036do { \
8037 struct btrfs_block_rsv *__rsv = &(fs_info)->rsv_name; \
8038 spin_lock(&__rsv->lock); \
8039 btrfs_info(fs_info, #rsv_name ": size %llu reserved %llu", \
8040 __rsv->size, __rsv->reserved); \
8041 spin_unlock(&__rsv->lock); \
8042} while (0)
8043
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008044static void dump_space_info(struct btrfs_fs_info *fs_info,
8045 struct btrfs_space_info *info, u64 bytes,
Josef Bacik9ed74f22009-09-11 16:12:44 -04008046 int dump_block_groups)
Josef Bacik0f9dd462008-09-23 13:14:11 -04008047{
8048 struct btrfs_block_group_cache *cache;
Yan, Zhengb742bb822010-05-16 10:46:24 -04008049 int index = 0;
Josef Bacik0f9dd462008-09-23 13:14:11 -04008050
Josef Bacik9ed74f22009-09-11 16:12:44 -04008051 spin_lock(&info->lock);
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008052 btrfs_info(fs_info, "space_info %llu has %llu free, is %sfull",
8053 info->flags,
Liu Bo41361352017-02-13 15:42:21 -08008054 info->total_bytes - btrfs_space_info_used(info, true),
8055 info->full ? "" : "not ");
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008056 btrfs_info(fs_info,
8057 "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu",
8058 info->total_bytes, info->bytes_used, info->bytes_pinned,
8059 info->bytes_reserved, info->bytes_may_use,
8060 info->bytes_readonly);
Josef Bacik9ed74f22009-09-11 16:12:44 -04008061 spin_unlock(&info->lock);
8062
Josef Bacikb78e5612018-11-21 14:03:07 -05008063 DUMP_BLOCK_RSV(fs_info, global_block_rsv);
8064 DUMP_BLOCK_RSV(fs_info, trans_block_rsv);
8065 DUMP_BLOCK_RSV(fs_info, chunk_block_rsv);
8066 DUMP_BLOCK_RSV(fs_info, delayed_block_rsv);
8067 DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv);
8068
Josef Bacik9ed74f22009-09-11 16:12:44 -04008069 if (!dump_block_groups)
8070 return;
Josef Bacik0f9dd462008-09-23 13:14:11 -04008071
Josef Bacik80eb2342008-10-29 14:49:05 -04008072 down_read(&info->groups_sem);
Yan, Zhengb742bb822010-05-16 10:46:24 -04008073again:
8074 list_for_each_entry(cache, &info->block_groups[index], list) {
Josef Bacik0f9dd462008-09-23 13:14:11 -04008075 spin_lock(&cache->lock);
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008076 btrfs_info(fs_info,
8077 "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s",
8078 cache->key.objectid, cache->key.offset,
8079 btrfs_block_group_used(&cache->item), cache->pinned,
8080 cache->reserved, cache->ro ? "[readonly]" : "");
Josef Bacik0f9dd462008-09-23 13:14:11 -04008081 btrfs_dump_free_space(cache, bytes);
8082 spin_unlock(&cache->lock);
8083 }
Yan, Zhengb742bb822010-05-16 10:46:24 -04008084 if (++index < BTRFS_NR_RAID_TYPES)
8085 goto again;
Josef Bacik80eb2342008-10-29 14:49:05 -04008086 up_read(&info->groups_sem);
Josef Bacik0f9dd462008-09-23 13:14:11 -04008087}
Zheng Yane8569812008-09-26 10:05:48 -04008088
Nikolay Borisov6f47c702018-03-13 12:22:32 +02008089/*
8090 * btrfs_reserve_extent - entry point to the extent allocator. Tries to find a
8091 * hole that is at least as big as @num_bytes.
8092 *
8093 * @root - The root that will contain this extent
8094 *
8095 * @ram_bytes - The amount of space in ram that @num_bytes take. This
8096 * is used for accounting purposes. This value differs
8097 * from @num_bytes only in the case of compressed extents.
8098 *
8099 * @num_bytes - Number of bytes to allocate on-disk.
8100 *
8101 * @min_alloc_size - Indicates the minimum amount of space that the
8102 * allocator should try to satisfy. In some cases
8103 * @num_bytes may be larger than what is required and if
8104 * the filesystem is fragmented then allocation fails.
8105 * However, the presence of @min_alloc_size gives a
8106 * chance to try and satisfy the smaller allocation.
8107 *
8108 * @empty_size - A hint that you plan on doing more COW. This is the
8109 * size in bytes the allocator should try to find free
8110 * next to the block it returns. This is just a hint and
8111 * may be ignored by the allocator.
8112 *
8113 * @hint_byte - Hint to the allocator to start searching above the byte
8114 * address passed. It might be ignored.
8115 *
8116 * @ins - This key is modified to record the found hole. It will
8117 * have the following values:
8118 * ins->objectid == start position
8119 * ins->flags = BTRFS_EXTENT_ITEM_KEY
8120 * ins->offset == the size of the hole.
8121 *
8122 * @is_data - Boolean flag indicating whether an extent is
8123 * allocated for data (true) or metadata (false)
8124 *
8125 * @delalloc - Boolean flag indicating whether this allocation is for
8126 * delalloc or not. If 'true' data_rwsem of block groups
8127 * is going to be acquired.
8128 *
8129 *
8130 * Returns 0 when an allocation succeeded or < 0 when an error occurred. In
8131 * case -ENOSPC is returned then @ins->offset will contain the size of the
8132 * largest available hole the allocator managed to find.
8133 */
Wang Xiaoguang18513092016-07-25 15:51:40 +08008134int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes,
Yan Zheng11833d62009-09-11 16:11:19 -04008135 u64 num_bytes, u64 min_alloc_size,
8136 u64 empty_size, u64 hint_byte,
Miao Xiee570fd22014-06-19 10:42:50 +08008137 struct btrfs_key *ins, int is_data, int delalloc)
Chris Masonfec577f2007-02-26 10:40:21 -05008138{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008139 struct btrfs_fs_info *fs_info = root->fs_info;
Josef Bacik36af4e02015-09-25 16:13:11 -04008140 bool final_tried = num_bytes == min_alloc_size;
David Sterbab6919a52013-04-29 13:39:40 +00008141 u64 flags;
Chris Masonfec577f2007-02-26 10:40:21 -05008142 int ret;
Chris Mason925baed2008-06-25 16:01:30 -04008143
Jeff Mahoney1b868262017-05-17 11:38:35 -04008144 flags = get_alloc_profile_by_root(root, is_data);
Chris Mason98d20f62008-04-14 09:46:10 -04008145again:
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008146 WARN_ON(num_bytes < fs_info->sectorsize);
Jeff Mahoney87bde3c2017-02-15 16:28:27 -05008147 ret = find_free_extent(fs_info, ram_bytes, num_bytes, empty_size,
Wang Xiaoguang18513092016-07-25 15:51:40 +08008148 hint_byte, ins, flags, delalloc);
Filipe Manana9cfa3e32016-04-26 15:39:32 +01008149 if (!ret && !is_data) {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008150 btrfs_dec_block_group_reservations(fs_info, ins->objectid);
Filipe Manana9cfa3e32016-04-26 15:39:32 +01008151 } else if (ret == -ENOSPC) {
Miao Xiea4820392013-09-09 13:19:42 +08008152 if (!final_tried && ins->offset) {
8153 num_bytes = min(num_bytes >> 1, ins->offset);
Jeff Mahoneyda170662016-06-15 09:22:56 -04008154 num_bytes = round_down(num_bytes,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008155 fs_info->sectorsize);
Miao Xie9e622d62012-01-26 15:01:12 -05008156 num_bytes = max(num_bytes, min_alloc_size);
Wang Xiaoguang18513092016-07-25 15:51:40 +08008157 ram_bytes = num_bytes;
Miao Xie9e622d62012-01-26 15:01:12 -05008158 if (num_bytes == min_alloc_size)
8159 final_tried = true;
8160 goto again;
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008161 } else if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
Miao Xie9e622d62012-01-26 15:01:12 -05008162 struct btrfs_space_info *sinfo;
Josef Bacik0f9dd462008-09-23 13:14:11 -04008163
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008164 sinfo = __find_space_info(fs_info, flags);
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008165 btrfs_err(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04008166 "allocation failed flags %llu, wanted %llu",
8167 flags, num_bytes);
Jeff Mahoney53804282012-03-01 14:56:28 +01008168 if (sinfo)
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04008169 dump_space_info(fs_info, sinfo, num_bytes, 1);
Miao Xie9e622d62012-01-26 15:01:12 -05008170 }
Chris Mason925baed2008-06-25 16:01:30 -04008171 }
Josef Bacik0f9dd462008-09-23 13:14:11 -04008172
8173 return ret;
Chris Masone6dcd2d2008-07-17 12:53:50 -04008174}
8175
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008176static int __btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info,
Miao Xiee570fd22014-06-19 10:42:50 +08008177 u64 start, u64 len,
8178 int pin, int delalloc)
Chris Mason65b51a02008-08-01 15:11:20 -04008179{
Josef Bacik0f9dd462008-09-23 13:14:11 -04008180 struct btrfs_block_group_cache *cache;
Liu Hui1f3c79a2009-01-05 15:57:51 -05008181 int ret = 0;
Josef Bacik0f9dd462008-09-23 13:14:11 -04008182
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008183 cache = btrfs_lookup_block_group(fs_info, start);
Josef Bacik0f9dd462008-09-23 13:14:11 -04008184 if (!cache) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008185 btrfs_err(fs_info, "Unable to find block group for %llu",
8186 start);
Josef Bacik0f9dd462008-09-23 13:14:11 -04008187 return -ENOSPC;
8188 }
Liu Hui1f3c79a2009-01-05 15:57:51 -05008189
Chris Masone688b7252011-10-31 20:52:39 -04008190 if (pin)
David Sterbafdf08602019-03-20 12:12:32 +01008191 pin_down_extent(cache, start, len, 1);
Chris Masone688b7252011-10-31 20:52:39 -04008192 else {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008193 if (btrfs_test_opt(fs_info, DISCARD))
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008194 ret = btrfs_discard_extent(fs_info, start, len, NULL);
Chris Masone688b7252011-10-31 20:52:39 -04008195 btrfs_add_free_space(cache, start, len);
Wang Xiaoguang4824f1f2016-07-25 15:51:39 +08008196 btrfs_free_reserved_bytes(cache, len, delalloc);
Jeff Mahoney71ff6432016-09-06 16:00:42 -04008197 trace_btrfs_reserved_extent_free(fs_info, start, len);
Chris Masone688b7252011-10-31 20:52:39 -04008198 }
Dongsheng Yang31193212014-12-12 16:44:35 +08008199
Chris Masonfa9c0d792009-04-03 09:47:43 -04008200 btrfs_put_block_group(cache);
Chris Masone6dcd2d2008-07-17 12:53:50 -04008201 return ret;
8202}
8203
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008204int btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info,
Miao Xiee570fd22014-06-19 10:42:50 +08008205 u64 start, u64 len, int delalloc)
Chris Masone688b7252011-10-31 20:52:39 -04008206{
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008207 return __btrfs_free_reserved_extent(fs_info, start, len, 0, delalloc);
Chris Masone688b7252011-10-31 20:52:39 -04008208}
8209
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008210int btrfs_free_and_pin_reserved_extent(struct btrfs_fs_info *fs_info,
Chris Masone688b7252011-10-31 20:52:39 -04008211 u64 start, u64 len)
8212{
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008213 return __btrfs_free_reserved_extent(fs_info, start, len, 1, 0);
Chris Masone688b7252011-10-31 20:52:39 -04008214}
8215
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008216static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008217 u64 parent, u64 root_objectid,
8218 u64 flags, u64 owner, u64 offset,
8219 struct btrfs_key *ins, int ref_mod)
Chris Masone6dcd2d2008-07-17 12:53:50 -04008220{
Nikolay Borisovef89b822018-06-20 15:48:58 +03008221 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Masone6dcd2d2008-07-17 12:53:50 -04008222 int ret;
Chris Masone6dcd2d2008-07-17 12:53:50 -04008223 struct btrfs_extent_item *extent_item;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008224 struct btrfs_extent_inline_ref *iref;
Chris Masone6dcd2d2008-07-17 12:53:50 -04008225 struct btrfs_path *path;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008226 struct extent_buffer *leaf;
8227 int type;
8228 u32 size;
Chris Masonf2654de2007-06-26 12:20:46 -04008229
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008230 if (parent > 0)
8231 type = BTRFS_SHARED_DATA_REF_KEY;
8232 else
8233 type = BTRFS_EXTENT_DATA_REF_KEY;
Zheng Yan31840ae2008-09-23 13:14:14 -04008234
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008235 size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
Chris Mason7bb86312007-12-11 09:25:06 -05008236
8237 path = btrfs_alloc_path();
Tsutomu Itohdb5b4932011-03-23 08:14:16 +00008238 if (!path)
8239 return -ENOMEM;
Chris Mason47e4bb92008-02-01 14:51:59 -05008240
Chris Masonb9473432009-03-13 11:00:37 -04008241 path->leave_spinning = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008242 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
8243 ins, size);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008244 if (ret) {
8245 btrfs_free_path(path);
8246 return ret;
8247 }
Josef Bacik0f9dd462008-09-23 13:14:11 -04008248
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008249 leaf = path->nodes[0];
8250 extent_item = btrfs_item_ptr(leaf, path->slots[0],
Chris Mason47e4bb92008-02-01 14:51:59 -05008251 struct btrfs_extent_item);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008252 btrfs_set_extent_refs(leaf, extent_item, ref_mod);
8253 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
8254 btrfs_set_extent_flags(leaf, extent_item,
8255 flags | BTRFS_EXTENT_FLAG_DATA);
Chris Mason47e4bb92008-02-01 14:51:59 -05008256
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008257 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
8258 btrfs_set_extent_inline_ref_type(leaf, iref, type);
8259 if (parent > 0) {
8260 struct btrfs_shared_data_ref *ref;
8261 ref = (struct btrfs_shared_data_ref *)(iref + 1);
8262 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
8263 btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
8264 } else {
8265 struct btrfs_extent_data_ref *ref;
8266 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
8267 btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
8268 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
8269 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
8270 btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
8271 }
Chris Mason47e4bb92008-02-01 14:51:59 -05008272
8273 btrfs_mark_buffer_dirty(path->nodes[0]);
Chris Mason7bb86312007-12-11 09:25:06 -05008274 btrfs_free_path(path);
Chris Masonf510cfe2007-10-15 16:14:48 -04008275
Nikolay Borisov25a356d2018-05-10 15:44:54 +03008276 ret = remove_from_free_space_tree(trans, ins->objectid, ins->offset);
Omar Sandoval1e144fb2015-09-29 20:50:37 -07008277 if (ret)
8278 return ret;
8279
David Sterba6b279402019-03-20 12:10:15 +01008280 ret = update_block_group(trans, ins->objectid, ins->offset, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008281 if (ret) { /* -ENOENT, logic error */
Simon Kirbyc2cf52e2013-03-19 22:41:23 +00008282 btrfs_err(fs_info, "update block group failed for %llu %llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02008283 ins->objectid, ins->offset);
Chris Masonf5947062008-02-04 10:10:13 -05008284 BUG();
8285 }
Jeff Mahoney71ff6432016-09-06 16:00:42 -04008286 trace_btrfs_reserved_extent_alloc(fs_info, ins->objectid, ins->offset);
Chris Masone6dcd2d2008-07-17 12:53:50 -04008287 return ret;
8288}
8289
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008290static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008291 struct btrfs_delayed_ref_node *node,
Nikolay Borisov21ebfbe2018-05-21 12:27:22 +03008292 struct btrfs_delayed_extent_op *extent_op)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008293{
Nikolay Borisov9dcdbe02018-05-21 12:27:20 +03008294 struct btrfs_fs_info *fs_info = trans->fs_info;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008295 int ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008296 struct btrfs_extent_item *extent_item;
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008297 struct btrfs_key extent_key;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008298 struct btrfs_tree_block_info *block_info;
8299 struct btrfs_extent_inline_ref *iref;
8300 struct btrfs_path *path;
8301 struct extent_buffer *leaf;
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008302 struct btrfs_delayed_tree_ref *ref;
Josef Bacik3173a182013-03-07 14:22:04 -05008303 u32 size = sizeof(*extent_item) + sizeof(*iref);
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008304 u64 num_bytes;
Nikolay Borisov21ebfbe2018-05-21 12:27:22 +03008305 u64 flags = extent_op->flags_to_set;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008306 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA);
Josef Bacik3173a182013-03-07 14:22:04 -05008307
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008308 ref = btrfs_delayed_node_to_tree_ref(node);
8309
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008310 extent_key.objectid = node->bytenr;
8311 if (skinny_metadata) {
8312 extent_key.offset = ref->level;
8313 extent_key.type = BTRFS_METADATA_ITEM_KEY;
8314 num_bytes = fs_info->nodesize;
8315 } else {
8316 extent_key.offset = node->num_bytes;
8317 extent_key.type = BTRFS_EXTENT_ITEM_KEY;
Josef Bacik3173a182013-03-07 14:22:04 -05008318 size += sizeof(*block_info);
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008319 num_bytes = node->num_bytes;
8320 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008321
8322 path = btrfs_alloc_path();
Josef Bacik80ee54b2018-10-11 15:54:22 -04008323 if (!path)
Mark Fashehd8926bb2011-07-13 10:38:47 -07008324 return -ENOMEM;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008325
8326 path->leave_spinning = 1;
8327 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008328 &extent_key, size);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008329 if (ret) {
Chris Masondd825252015-04-01 08:36:05 -07008330 btrfs_free_path(path);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008331 return ret;
8332 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008333
8334 leaf = path->nodes[0];
8335 extent_item = btrfs_item_ptr(leaf, path->slots[0],
8336 struct btrfs_extent_item);
8337 btrfs_set_extent_refs(leaf, extent_item, 1);
8338 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
8339 btrfs_set_extent_flags(leaf, extent_item,
8340 flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008341
Josef Bacik3173a182013-03-07 14:22:04 -05008342 if (skinny_metadata) {
8343 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
8344 } else {
8345 block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
Nikolay Borisov21ebfbe2018-05-21 12:27:22 +03008346 btrfs_set_tree_block_key(leaf, block_info, &extent_op->key);
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008347 btrfs_set_tree_block_level(leaf, block_info, ref->level);
Josef Bacik3173a182013-03-07 14:22:04 -05008348 iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
8349 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008350
Nikolay Borisovd4b20732018-05-21 12:27:23 +03008351 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008352 BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
8353 btrfs_set_extent_inline_ref_type(leaf, iref,
8354 BTRFS_SHARED_BLOCK_REF_KEY);
Nikolay Borisovd4b20732018-05-21 12:27:23 +03008355 btrfs_set_extent_inline_ref_offset(leaf, iref, ref->parent);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008356 } else {
8357 btrfs_set_extent_inline_ref_type(leaf, iref,
8358 BTRFS_TREE_BLOCK_REF_KEY);
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008359 btrfs_set_extent_inline_ref_offset(leaf, iref, ref->root);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008360 }
8361
8362 btrfs_mark_buffer_dirty(leaf);
8363 btrfs_free_path(path);
8364
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008365 ret = remove_from_free_space_tree(trans, extent_key.objectid,
8366 num_bytes);
Omar Sandoval1e144fb2015-09-29 20:50:37 -07008367 if (ret)
8368 return ret;
8369
David Sterba6b279402019-03-20 12:10:15 +01008370 ret = update_block_group(trans, extent_key.objectid,
Jeff Mahoney6202df62016-06-22 18:54:22 -04008371 fs_info->nodesize, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008372 if (ret) { /* -ENOENT, logic error */
Simon Kirbyc2cf52e2013-03-19 22:41:23 +00008373 btrfs_err(fs_info, "update block group failed for %llu %llu",
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008374 extent_key.objectid, extent_key.offset);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008375 BUG();
8376 }
Josef Bacik0be5dc62013-10-07 15:18:52 -04008377
Nikolay Borisov4e6bd4e2018-05-21 12:27:21 +03008378 trace_btrfs_reserved_extent_alloc(fs_info, extent_key.objectid,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008379 fs_info->nodesize);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008380 return ret;
8381}
8382
8383int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
Josef Bacik84f7d8e2017-09-29 15:43:49 -04008384 struct btrfs_root *root, u64 owner,
Qu Wenruo5846a3c2015-10-26 14:11:18 +08008385 u64 offset, u64 ram_bytes,
8386 struct btrfs_key *ins)
Chris Masone6dcd2d2008-07-17 12:53:50 -04008387{
8388 int ret;
Chris Mason1c2308f82008-09-23 13:14:13 -04008389
Josef Bacik84f7d8e2017-09-29 15:43:49 -04008390 BUG_ON(root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID);
Chris Mason56bec292009-03-13 10:10:06 -04008391
Josef Bacikfd708b82017-09-29 15:43:50 -04008392 btrfs_ref_tree_mod(root, ins->objectid, ins->offset, 0,
8393 root->root_key.objectid, owner, offset,
8394 BTRFS_ADD_DELAYED_EXTENT);
8395
Nikolay Borisov88a979c2018-06-20 15:48:54 +03008396 ret = btrfs_add_delayed_data_ref(trans, ins->objectid,
Josef Bacik84f7d8e2017-09-29 15:43:49 -04008397 ins->offset, 0,
8398 root->root_key.objectid, owner,
Omar Sandoval7be07912017-06-06 16:45:30 -07008399 offset, ram_bytes,
8400 BTRFS_ADD_DELAYED_EXTENT, NULL, NULL);
Chris Masone6dcd2d2008-07-17 12:53:50 -04008401 return ret;
8402}
Chris Masone02119d2008-09-05 16:13:11 -04008403
8404/*
8405 * this is used by the tree logging recovery code. It records that
8406 * an extent has been allocated and makes sure to clear the free
8407 * space cache bits as well
8408 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008409int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04008410 u64 root_objectid, u64 owner, u64 offset,
8411 struct btrfs_key *ins)
Chris Masone02119d2008-09-05 16:13:11 -04008412{
Nikolay Borisov61da2ab2018-06-20 15:49:13 +03008413 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Masone02119d2008-09-05 16:13:11 -04008414 int ret;
8415 struct btrfs_block_group_cache *block_group;
Wang Xiaoguanged7a6942016-08-26 11:33:14 +08008416 struct btrfs_space_info *space_info;
Josef Bacik8c2a1a32013-06-06 13:19:32 -04008417
8418 /*
8419 * Mixed block groups will exclude before processing the log so we only
Nicholas D Steeves01327612016-05-19 21:18:45 -04008420 * need to do the exclude dance if this fs isn't mixed.
Josef Bacik8c2a1a32013-06-06 13:19:32 -04008421 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008422 if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS)) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008423 ret = __exclude_logged_extent(fs_info, ins->objectid,
8424 ins->offset);
Josef Bacik8c2a1a32013-06-06 13:19:32 -04008425 if (ret)
8426 return ret;
8427 }
Chris Masone02119d2008-09-05 16:13:11 -04008428
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008429 block_group = btrfs_lookup_block_group(fs_info, ins->objectid);
Josef Bacik8c2a1a32013-06-06 13:19:32 -04008430 if (!block_group)
8431 return -EINVAL;
Yan Zheng11833d62009-09-11 16:11:19 -04008432
Wang Xiaoguanged7a6942016-08-26 11:33:14 +08008433 space_info = block_group->space_info;
8434 spin_lock(&space_info->lock);
8435 spin_lock(&block_group->lock);
8436 space_info->bytes_reserved += ins->offset;
8437 block_group->reserved += ins->offset;
8438 spin_unlock(&block_group->lock);
8439 spin_unlock(&space_info->lock);
8440
Nikolay Borisovef89b822018-06-20 15:48:58 +03008441 ret = alloc_reserved_file_extent(trans, 0, root_objectid, 0, owner,
8442 offset, ins, 1);
Josef Bacikb50c6e22013-04-25 15:55:30 -04008443 btrfs_put_block_group(block_group);
Chris Masone02119d2008-09-05 16:13:11 -04008444 return ret;
8445}
8446
Eric Sandeen48a3b632013-04-25 20:41:01 +00008447static struct extent_buffer *
8448btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root,
Nikolay Borisovbc877d22018-06-18 14:13:19 +03008449 u64 bytenr, int level, u64 owner)
Chris Mason65b51a02008-08-01 15:11:20 -04008450{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008451 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason65b51a02008-08-01 15:11:20 -04008452 struct extent_buffer *buf;
8453
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008454 buf = btrfs_find_create_tree_block(fs_info, bytenr);
Liu Boc871b0f2016-06-06 12:01:23 -07008455 if (IS_ERR(buf))
8456 return buf;
8457
Qu Wenruob72c3ab2018-08-21 09:53:47 +08008458 /*
8459 * Extra safety check in case the extent tree is corrupted and extent
8460 * allocator chooses to use a tree block which is already used and
8461 * locked.
8462 */
8463 if (buf->lock_owner == current->pid) {
8464 btrfs_err_rl(fs_info,
8465"tree block %llu owner %llu already locked by pid=%d, extent tree corruption detected",
8466 buf->start, btrfs_header_owner(buf), current->pid);
8467 free_extent_buffer(buf);
8468 return ERR_PTR(-EUCLEAN);
8469 }
8470
Chris Mason85d4e462011-07-26 16:11:19 -04008471 btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
Chris Mason65b51a02008-08-01 15:11:20 -04008472 btrfs_tree_lock(buf);
David Sterba6a884d7d2019-03-20 14:30:02 +01008473 btrfs_clean_tree_block(buf);
Josef Bacik3083ee22012-03-09 16:01:49 -05008474 clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
Chris Masonb4ce94d2009-02-04 09:25:08 -05008475
David Sterba8bead252018-04-04 02:03:48 +02008476 btrfs_set_lock_blocking_write(buf);
David Sterba4db8c522015-12-03 13:06:46 +01008477 set_extent_buffer_uptodate(buf);
Chris Masonb4ce94d2009-02-04 09:25:08 -05008478
Nikolay Borisovbc877d22018-06-18 14:13:19 +03008479 memzero_extent_buffer(buf, 0, sizeof(struct btrfs_header));
8480 btrfs_set_header_level(buf, level);
8481 btrfs_set_header_bytenr(buf, buf->start);
8482 btrfs_set_header_generation(buf, trans->transid);
8483 btrfs_set_header_backref_rev(buf, BTRFS_MIXED_BACKREF_REV);
8484 btrfs_set_header_owner(buf, owner);
Nikolay Borisovde37aa52018-10-30 16:43:24 +02008485 write_extent_buffer_fsid(buf, fs_info->fs_devices->metadata_uuid);
Nikolay Borisovbc877d22018-06-18 14:13:19 +03008486 write_extent_buffer_chunk_tree_uuid(buf, fs_info->chunk_tree_uuid);
Chris Masond0c803c2008-09-11 16:17:57 -04008487 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
Filipe Manana656f30d2014-09-26 12:25:56 +01008488 buf->log_index = root->log_transid % 2;
Yan, Zheng8cef4e12009-11-12 09:33:26 +00008489 /*
8490 * we allow two log transactions at a time, use different
Andrea Gelmini52042d82018-11-28 12:05:13 +01008491 * EXTENT bit to differentiate dirty pages.
Yan, Zheng8cef4e12009-11-12 09:33:26 +00008492 */
Filipe Manana656f30d2014-09-26 12:25:56 +01008493 if (buf->log_index == 0)
Yan, Zheng8cef4e12009-11-12 09:33:26 +00008494 set_extent_dirty(&root->dirty_log_pages, buf->start,
8495 buf->start + buf->len - 1, GFP_NOFS);
8496 else
8497 set_extent_new(&root->dirty_log_pages, buf->start,
David Sterba3744dbe2016-04-26 23:54:39 +02008498 buf->start + buf->len - 1);
Chris Masond0c803c2008-09-11 16:17:57 -04008499 } else {
Filipe Manana656f30d2014-09-26 12:25:56 +01008500 buf->log_index = -1;
Chris Masond0c803c2008-09-11 16:17:57 -04008501 set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
8502 buf->start + buf->len - 1, GFP_NOFS);
8503 }
Jeff Mahoney64c12922016-06-08 00:36:38 -04008504 trans->dirty = true;
Chris Masonb4ce94d2009-02-04 09:25:08 -05008505 /* this returns a buffer locked for blocking */
Chris Mason65b51a02008-08-01 15:11:20 -04008506 return buf;
8507}
8508
Yan, Zhengf0486c62010-05-16 10:46:25 -04008509static struct btrfs_block_rsv *
8510use_block_rsv(struct btrfs_trans_handle *trans,
8511 struct btrfs_root *root, u32 blocksize)
8512{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008513 struct btrfs_fs_info *fs_info = root->fs_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04008514 struct btrfs_block_rsv *block_rsv;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008515 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
Yan, Zhengf0486c62010-05-16 10:46:25 -04008516 int ret;
Miao Xied88033d2013-05-13 13:55:12 +00008517 bool global_updated = false;
Yan, Zhengf0486c62010-05-16 10:46:25 -04008518
8519 block_rsv = get_block_rsv(trans, root);
8520
Miao Xieb586b322013-05-13 13:55:10 +00008521 if (unlikely(block_rsv->size == 0))
8522 goto try_reserve;
Miao Xied88033d2013-05-13 13:55:12 +00008523again:
Yan, Zhengf0486c62010-05-16 10:46:25 -04008524 ret = block_rsv_use_bytes(block_rsv, blocksize);
8525 if (!ret)
8526 return block_rsv;
8527
Miao Xieb586b322013-05-13 13:55:10 +00008528 if (block_rsv->failfast)
8529 return ERR_PTR(ret);
8530
Miao Xied88033d2013-05-13 13:55:12 +00008531 if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) {
8532 global_updated = true;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008533 update_global_block_rsv(fs_info);
Miao Xied88033d2013-05-13 13:55:12 +00008534 goto again;
8535 }
8536
Josef Bacikba2c4d42018-12-03 10:20:33 -05008537 /*
8538 * The global reserve still exists to save us from ourselves, so don't
8539 * warn_on if we are short on our delayed refs reserve.
8540 */
8541 if (block_rsv->type != BTRFS_BLOCK_RSV_DELREFS &&
8542 btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
Miao Xieb586b322013-05-13 13:55:10 +00008543 static DEFINE_RATELIMIT_STATE(_rs,
8544 DEFAULT_RATELIMIT_INTERVAL * 10,
8545 /*DEFAULT_RATELIMIT_BURST*/ 1);
8546 if (__ratelimit(&_rs))
8547 WARN(1, KERN_DEBUG
Frank Holtonefe120a2013-12-20 11:37:06 -05008548 "BTRFS: block rsv returned %d\n", ret);
Miao Xieb586b322013-05-13 13:55:10 +00008549 }
8550try_reserve:
8551 ret = reserve_metadata_bytes(root, block_rsv, blocksize,
8552 BTRFS_RESERVE_NO_FLUSH);
8553 if (!ret)
8554 return block_rsv;
8555 /*
8556 * If we couldn't reserve metadata bytes try and use some from
Miao Xie5881cfc2013-05-13 13:55:11 +00008557 * the global reserve if its space type is the same as the global
8558 * reservation.
Miao Xieb586b322013-05-13 13:55:10 +00008559 */
Miao Xie5881cfc2013-05-13 13:55:11 +00008560 if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL &&
8561 block_rsv->space_info == global_rsv->space_info) {
Miao Xieb586b322013-05-13 13:55:10 +00008562 ret = block_rsv_use_bytes(global_rsv, blocksize);
8563 if (!ret)
8564 return global_rsv;
8565 }
8566 return ERR_PTR(ret);
Yan, Zhengf0486c62010-05-16 10:46:25 -04008567}
8568
Josef Bacik8c2a3ca2012-01-10 10:31:31 -05008569static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
8570 struct btrfs_block_rsv *block_rsv, u32 blocksize)
Yan, Zhengf0486c62010-05-16 10:46:25 -04008571{
Lu Fengqi3a584172018-08-04 21:10:55 +08008572 block_rsv_add_bytes(block_rsv, blocksize, false);
Qu Wenruoff6bc372017-12-21 13:42:04 +08008573 block_rsv_release_bytes(fs_info, block_rsv, NULL, 0, NULL);
Yan, Zhengf0486c62010-05-16 10:46:25 -04008574}
8575
Chris Masonfec577f2007-02-26 10:40:21 -05008576/*
Yan, Zhengf0486c62010-05-16 10:46:25 -04008577 * finds a free extent and does all the dirty work required for allocation
Omar Sandoval67b78592015-02-24 02:47:04 -08008578 * returns the tree buffer or an ERR_PTR on error.
Chris Masonfec577f2007-02-26 10:40:21 -05008579 */
David Sterba4d75f8a2014-06-15 01:54:12 +02008580struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans,
Omar Sandoval310712b2017-01-17 23:24:37 -08008581 struct btrfs_root *root,
8582 u64 parent, u64 root_objectid,
8583 const struct btrfs_disk_key *key,
8584 int level, u64 hint,
8585 u64 empty_size)
Chris Masonfec577f2007-02-26 10:40:21 -05008586{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008587 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Masone2fa7222007-03-12 16:22:34 -04008588 struct btrfs_key ins;
Yan, Zhengf0486c62010-05-16 10:46:25 -04008589 struct btrfs_block_rsv *block_rsv;
Chris Mason5f39d392007-10-15 16:14:19 -04008590 struct extent_buffer *buf;
Omar Sandoval67b78592015-02-24 02:47:04 -08008591 struct btrfs_delayed_extent_op *extent_op;
Qu Wenruoed4f2552019-04-04 14:45:31 +08008592 struct btrfs_ref generic_ref = { 0 };
Yan, Zhengf0486c62010-05-16 10:46:25 -04008593 u64 flags = 0;
8594 int ret;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008595 u32 blocksize = fs_info->nodesize;
8596 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA);
Yan, Zhengf0486c62010-05-16 10:46:25 -04008597
David Sterba05653ef2016-07-15 15:23:37 +02008598#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008599 if (btrfs_is_testing(fs_info)) {
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04008600 buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr,
Nikolay Borisovbc877d22018-06-18 14:13:19 +03008601 level, root_objectid);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04008602 if (!IS_ERR(buf))
8603 root->alloc_bytenr += blocksize;
8604 return buf;
8605 }
David Sterba05653ef2016-07-15 15:23:37 +02008606#endif
David Sterbafccb84c2014-09-29 23:53:21 +02008607
Yan, Zhengf0486c62010-05-16 10:46:25 -04008608 block_rsv = use_block_rsv(trans, root, blocksize);
8609 if (IS_ERR(block_rsv))
8610 return ERR_CAST(block_rsv);
8611
Wang Xiaoguang18513092016-07-25 15:51:40 +08008612 ret = btrfs_reserve_extent(root, blocksize, blocksize, blocksize,
Miao Xiee570fd22014-06-19 10:42:50 +08008613 empty_size, hint, &ins, 0, 0);
Omar Sandoval67b78592015-02-24 02:47:04 -08008614 if (ret)
8615 goto out_unuse;
Chris Mason55c69072008-01-09 15:55:33 -05008616
Nikolay Borisovbc877d22018-06-18 14:13:19 +03008617 buf = btrfs_init_new_buffer(trans, root, ins.objectid, level,
8618 root_objectid);
Omar Sandoval67b78592015-02-24 02:47:04 -08008619 if (IS_ERR(buf)) {
8620 ret = PTR_ERR(buf);
8621 goto out_free_reserved;
8622 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04008623
8624 if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
8625 if (parent == 0)
8626 parent = ins.objectid;
8627 flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
8628 } else
8629 BUG_ON(parent > 0);
8630
8631 if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
Miao Xie78a61842012-11-21 02:21:28 +00008632 extent_op = btrfs_alloc_delayed_extent_op();
Omar Sandoval67b78592015-02-24 02:47:04 -08008633 if (!extent_op) {
8634 ret = -ENOMEM;
8635 goto out_free_buf;
8636 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04008637 if (key)
8638 memcpy(&extent_op->key, key, sizeof(extent_op->key));
8639 else
8640 memset(&extent_op->key, 0, sizeof(extent_op->key));
8641 extent_op->flags_to_set = flags;
David Sterba35b3ad52015-11-30 16:51:29 +01008642 extent_op->update_key = skinny_metadata ? false : true;
8643 extent_op->update_flags = true;
8644 extent_op->is_data = false;
Josef Bacikb1c79e02013-05-09 13:49:30 -04008645 extent_op->level = level;
Yan, Zhengf0486c62010-05-16 10:46:25 -04008646
Josef Bacikfd708b82017-09-29 15:43:50 -04008647 btrfs_ref_tree_mod(root, ins.objectid, ins.offset, parent,
8648 root_objectid, level, 0,
8649 BTRFS_ADD_DELAYED_EXTENT);
Qu Wenruoed4f2552019-04-04 14:45:31 +08008650 btrfs_init_generic_ref(&generic_ref, BTRFS_ADD_DELAYED_EXTENT,
8651 ins.objectid, ins.offset, parent);
8652 generic_ref.real_root = root->root_key.objectid;
8653 btrfs_init_tree_ref(&generic_ref, level, root_objectid);
8654 ret = btrfs_add_delayed_tree_ref(trans, &generic_ref,
Omar Sandoval7be07912017-06-06 16:45:30 -07008655 extent_op, NULL, NULL);
Omar Sandoval67b78592015-02-24 02:47:04 -08008656 if (ret)
8657 goto out_free_delayed;
Yan, Zhengf0486c62010-05-16 10:46:25 -04008658 }
Chris Masonfec577f2007-02-26 10:40:21 -05008659 return buf;
Omar Sandoval67b78592015-02-24 02:47:04 -08008660
8661out_free_delayed:
8662 btrfs_free_delayed_extent_op(extent_op);
8663out_free_buf:
8664 free_extent_buffer(buf);
8665out_free_reserved:
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008666 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 0);
Omar Sandoval67b78592015-02-24 02:47:04 -08008667out_unuse:
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008668 unuse_block_rsv(fs_info, block_rsv, blocksize);
Omar Sandoval67b78592015-02-24 02:47:04 -08008669 return ERR_PTR(ret);
Chris Masonfec577f2007-02-26 10:40:21 -05008670}
Chris Masona28ec192007-03-06 20:08:01 -05008671
Yan Zheng2c47e6052009-06-27 21:07:35 -04008672struct walk_control {
8673 u64 refs[BTRFS_MAX_LEVEL];
8674 u64 flags[BTRFS_MAX_LEVEL];
8675 struct btrfs_key update_progress;
Josef Bacikaea6f022019-02-06 15:46:15 -05008676 struct btrfs_key drop_progress;
8677 int drop_level;
Yan Zheng2c47e6052009-06-27 21:07:35 -04008678 int stage;
8679 int level;
8680 int shared_level;
8681 int update_ref;
8682 int keep_locks;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008683 int reada_slot;
8684 int reada_count;
Josef Bacik78c52d92019-02-06 15:46:14 -05008685 int restarted;
Yan Zheng2c47e6052009-06-27 21:07:35 -04008686};
8687
8688#define DROP_REFERENCE 1
8689#define UPDATE_BACKREF 2
8690
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008691static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
8692 struct btrfs_root *root,
8693 struct walk_control *wc,
8694 struct btrfs_path *path)
8695{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008696 struct btrfs_fs_info *fs_info = root->fs_info;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008697 u64 bytenr;
8698 u64 generation;
8699 u64 refs;
Yan, Zheng94fcca92009-10-09 09:25:16 -04008700 u64 flags;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008701 u32 nritems;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008702 struct btrfs_key key;
8703 struct extent_buffer *eb;
8704 int ret;
8705 int slot;
8706 int nread = 0;
8707
8708 if (path->slots[wc->level] < wc->reada_slot) {
8709 wc->reada_count = wc->reada_count * 2 / 3;
8710 wc->reada_count = max(wc->reada_count, 2);
8711 } else {
8712 wc->reada_count = wc->reada_count * 3 / 2;
8713 wc->reada_count = min_t(int, wc->reada_count,
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008714 BTRFS_NODEPTRS_PER_BLOCK(fs_info));
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008715 }
8716
8717 eb = path->nodes[wc->level];
8718 nritems = btrfs_header_nritems(eb);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008719
8720 for (slot = path->slots[wc->level]; slot < nritems; slot++) {
8721 if (nread >= wc->reada_count)
8722 break;
8723
8724 cond_resched();
8725 bytenr = btrfs_node_blockptr(eb, slot);
8726 generation = btrfs_node_ptr_generation(eb, slot);
8727
8728 if (slot == path->slots[wc->level])
8729 goto reada;
8730
8731 if (wc->stage == UPDATE_BACKREF &&
8732 generation <= root->root_key.offset)
8733 continue;
8734
Yan, Zheng94fcca92009-10-09 09:25:16 -04008735 /* We don't lock the tree block, it's OK to be racy here */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008736 ret = btrfs_lookup_extent_info(trans, fs_info, bytenr,
Josef Bacik3173a182013-03-07 14:22:04 -05008737 wc->level - 1, 1, &refs,
8738 &flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008739 /* We don't care about errors in readahead. */
8740 if (ret < 0)
8741 continue;
Yan, Zheng94fcca92009-10-09 09:25:16 -04008742 BUG_ON(refs == 0);
8743
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008744 if (wc->stage == DROP_REFERENCE) {
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008745 if (refs == 1)
8746 goto reada;
8747
Yan, Zheng94fcca92009-10-09 09:25:16 -04008748 if (wc->level == 1 &&
8749 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8750 continue;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008751 if (!wc->update_ref ||
8752 generation <= root->root_key.offset)
8753 continue;
8754 btrfs_node_key_to_cpu(eb, &key, slot);
8755 ret = btrfs_comp_cpu_keys(&key,
8756 &wc->update_progress);
8757 if (ret < 0)
8758 continue;
Yan, Zheng94fcca92009-10-09 09:25:16 -04008759 } else {
8760 if (wc->level == 1 &&
8761 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8762 continue;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008763 }
8764reada:
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008765 readahead_tree_block(fs_info, bytenr);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008766 nread++;
8767 }
8768 wc->reada_slot = slot;
8769}
8770
Chris Mason9aca1d52007-03-13 11:09:37 -04008771/*
Liu Bo2c016dc2012-12-26 15:32:17 +08008772 * helper to process tree block while walking down the tree.
Yan Zheng2c47e6052009-06-27 21:07:35 -04008773 *
Yan Zheng2c47e6052009-06-27 21:07:35 -04008774 * when wc->stage == UPDATE_BACKREF, this function updates
8775 * back refs for pointers in the block.
8776 *
8777 * NOTE: return value 1 means we should stop walking down.
Yan Zhengf82d02d2008-10-29 14:49:05 -04008778 */
Yan Zheng2c47e6052009-06-27 21:07:35 -04008779static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
8780 struct btrfs_root *root,
8781 struct btrfs_path *path,
Yan, Zheng94fcca92009-10-09 09:25:16 -04008782 struct walk_control *wc, int lookup_info)
Yan Zheng2c47e6052009-06-27 21:07:35 -04008783{
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008784 struct btrfs_fs_info *fs_info = root->fs_info;
Yan Zheng2c47e6052009-06-27 21:07:35 -04008785 int level = wc->level;
8786 struct extent_buffer *eb = path->nodes[level];
Yan Zheng2c47e6052009-06-27 21:07:35 -04008787 u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
8788 int ret;
8789
8790 if (wc->stage == UPDATE_BACKREF &&
8791 btrfs_header_owner(eb) != root->root_key.objectid)
8792 return 1;
8793
8794 /*
8795 * when reference count of tree block is 1, it won't increase
8796 * again. once full backref flag is set, we never clear it.
8797 */
Yan, Zheng94fcca92009-10-09 09:25:16 -04008798 if (lookup_info &&
8799 ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
8800 (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
Yan Zheng2c47e6052009-06-27 21:07:35 -04008801 BUG_ON(!path->locks[level]);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008802 ret = btrfs_lookup_extent_info(trans, fs_info,
Josef Bacik3173a182013-03-07 14:22:04 -05008803 eb->start, level, 1,
Yan Zheng2c47e6052009-06-27 21:07:35 -04008804 &wc->refs[level],
8805 &wc->flags[level]);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008806 BUG_ON(ret == -ENOMEM);
8807 if (ret)
8808 return ret;
Yan Zheng2c47e6052009-06-27 21:07:35 -04008809 BUG_ON(wc->refs[level] == 0);
8810 }
8811
Yan Zheng2c47e6052009-06-27 21:07:35 -04008812 if (wc->stage == DROP_REFERENCE) {
8813 if (wc->refs[level] > 1)
8814 return 1;
8815
8816 if (path->locks[level] && !wc->keep_locks) {
Chris Masonbd681512011-07-16 15:23:14 -04008817 btrfs_tree_unlock_rw(eb, path->locks[level]);
Yan Zheng2c47e6052009-06-27 21:07:35 -04008818 path->locks[level] = 0;
8819 }
8820 return 0;
8821 }
8822
8823 /* wc->stage == UPDATE_BACKREF */
8824 if (!(wc->flags[level] & flag)) {
8825 BUG_ON(!path->locks[level]);
Josef Bacike339a6b2014-07-02 10:54:25 -07008826 ret = btrfs_inc_ref(trans, root, eb, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008827 BUG_ON(ret); /* -ENOMEM */
Josef Bacike339a6b2014-07-02 10:54:25 -07008828 ret = btrfs_dec_ref(trans, root, eb, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008829 BUG_ON(ret); /* -ENOMEM */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008830 ret = btrfs_set_disk_extent_flags(trans, fs_info, eb->start,
Josef Bacikb1c79e02013-05-09 13:49:30 -04008831 eb->len, flag,
8832 btrfs_header_level(eb), 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008833 BUG_ON(ret); /* -ENOMEM */
Yan Zheng2c47e6052009-06-27 21:07:35 -04008834 wc->flags[level] |= flag;
8835 }
8836
8837 /*
8838 * the block is shared by multiple trees, so it's not good to
8839 * keep the tree lock
8840 */
8841 if (path->locks[level] && level > 0) {
Chris Masonbd681512011-07-16 15:23:14 -04008842 btrfs_tree_unlock_rw(eb, path->locks[level]);
Yan Zheng2c47e6052009-06-27 21:07:35 -04008843 path->locks[level] = 0;
8844 }
8845 return 0;
8846}
8847
8848/*
Josef Bacik78c52d92019-02-06 15:46:14 -05008849 * This is used to verify a ref exists for this root to deal with a bug where we
8850 * would have a drop_progress key that hadn't been updated properly.
8851 */
8852static int check_ref_exists(struct btrfs_trans_handle *trans,
8853 struct btrfs_root *root, u64 bytenr, u64 parent,
8854 int level)
8855{
8856 struct btrfs_path *path;
8857 struct btrfs_extent_inline_ref *iref;
8858 int ret;
8859
8860 path = btrfs_alloc_path();
8861 if (!path)
8862 return -ENOMEM;
8863
8864 ret = lookup_extent_backref(trans, path, &iref, bytenr,
8865 root->fs_info->nodesize, parent,
8866 root->root_key.objectid, level, 0);
8867 btrfs_free_path(path);
8868 if (ret == -ENOENT)
8869 return 0;
8870 if (ret < 0)
8871 return ret;
8872 return 1;
8873}
8874
8875/*
Liu Bo2c016dc2012-12-26 15:32:17 +08008876 * helper to process tree block pointer.
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008877 *
8878 * when wc->stage == DROP_REFERENCE, this function checks
8879 * reference count of the block pointed to. if the block
8880 * is shared and we need update back refs for the subtree
8881 * rooted at the block, this function changes wc->stage to
8882 * UPDATE_BACKREF. if the block is shared and there is no
8883 * need to update back, this function drops the reference
8884 * to the block.
8885 *
8886 * NOTE: return value 1 means we should stop walking down.
8887 */
8888static noinline int do_walk_down(struct btrfs_trans_handle *trans,
8889 struct btrfs_root *root,
8890 struct btrfs_path *path,
Yan, Zheng94fcca92009-10-09 09:25:16 -04008891 struct walk_control *wc, int *lookup_info)
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008892{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008893 struct btrfs_fs_info *fs_info = root->fs_info;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008894 u64 bytenr;
8895 u64 generation;
8896 u64 parent;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008897 struct btrfs_key key;
Qu Wenruo581c1762018-03-29 09:08:11 +08008898 struct btrfs_key first_key;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008899 struct extent_buffer *next;
8900 int level = wc->level;
8901 int reada = 0;
8902 int ret = 0;
Mark Fasheh11526512014-07-17 12:39:01 -07008903 bool need_account = false;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008904
8905 generation = btrfs_node_ptr_generation(path->nodes[level],
8906 path->slots[level]);
8907 /*
8908 * if the lower level block was created before the snapshot
8909 * was created, we know there is no need to update back refs
8910 * for the subtree
8911 */
8912 if (wc->stage == UPDATE_BACKREF &&
Yan, Zheng94fcca92009-10-09 09:25:16 -04008913 generation <= root->root_key.offset) {
8914 *lookup_info = 1;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008915 return 1;
Yan, Zheng94fcca92009-10-09 09:25:16 -04008916 }
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008917
8918 bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
Qu Wenruo581c1762018-03-29 09:08:11 +08008919 btrfs_node_key_to_cpu(path->nodes[level], &first_key,
8920 path->slots[level]);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008921
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008922 next = find_extent_buffer(fs_info, bytenr);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008923 if (!next) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008924 next = btrfs_find_create_tree_block(fs_info, bytenr);
Liu Boc871b0f2016-06-06 12:01:23 -07008925 if (IS_ERR(next))
8926 return PTR_ERR(next);
8927
Josef Bacikb2aaaa32013-07-05 17:05:38 -04008928 btrfs_set_buffer_lockdep_class(root->root_key.objectid, next,
8929 level - 1);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008930 reada = 1;
8931 }
8932 btrfs_tree_lock(next);
David Sterba8bead252018-04-04 02:03:48 +02008933 btrfs_set_lock_blocking_write(next);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008934
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04008935 ret = btrfs_lookup_extent_info(trans, fs_info, bytenr, level - 1, 1,
Yan, Zheng94fcca92009-10-09 09:25:16 -04008936 &wc->refs[level - 1],
8937 &wc->flags[level - 1]);
Josef Bacik48672682016-09-23 13:23:28 +02008938 if (ret < 0)
8939 goto out_unlock;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01008940
Simon Kirbyc2cf52e2013-03-19 22:41:23 +00008941 if (unlikely(wc->refs[level - 1] == 0)) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04008942 btrfs_err(fs_info, "Missing references.");
Josef Bacik48672682016-09-23 13:23:28 +02008943 ret = -EIO;
8944 goto out_unlock;
Simon Kirbyc2cf52e2013-03-19 22:41:23 +00008945 }
Yan, Zheng94fcca92009-10-09 09:25:16 -04008946 *lookup_info = 0;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008947
Yan, Zheng94fcca92009-10-09 09:25:16 -04008948 if (wc->stage == DROP_REFERENCE) {
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008949 if (wc->refs[level - 1] > 1) {
Mark Fasheh11526512014-07-17 12:39:01 -07008950 need_account = true;
Yan, Zheng94fcca92009-10-09 09:25:16 -04008951 if (level == 1 &&
8952 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8953 goto skip;
8954
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008955 if (!wc->update_ref ||
8956 generation <= root->root_key.offset)
8957 goto skip;
8958
8959 btrfs_node_key_to_cpu(path->nodes[level], &key,
8960 path->slots[level]);
8961 ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
8962 if (ret < 0)
8963 goto skip;
8964
8965 wc->stage = UPDATE_BACKREF;
8966 wc->shared_level = level - 1;
8967 }
Yan, Zheng94fcca92009-10-09 09:25:16 -04008968 } else {
8969 if (level == 1 &&
8970 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8971 goto skip;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008972 }
8973
Chris Masonb9fab912012-05-06 07:23:47 -04008974 if (!btrfs_buffer_uptodate(next, generation, 0)) {
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008975 btrfs_tree_unlock(next);
8976 free_extent_buffer(next);
8977 next = NULL;
Yan, Zheng94fcca92009-10-09 09:25:16 -04008978 *lookup_info = 1;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008979 }
8980
8981 if (!next) {
8982 if (reada && level == 1)
8983 reada_walk_down(trans, root, wc, path);
Qu Wenruo581c1762018-03-29 09:08:11 +08008984 next = read_tree_block(fs_info, bytenr, generation, level - 1,
8985 &first_key);
Liu Bo64c043d2015-05-25 17:30:15 +08008986 if (IS_ERR(next)) {
8987 return PTR_ERR(next);
8988 } else if (!extent_buffer_uptodate(next)) {
Josef Bacik416bc652013-04-23 14:17:42 -04008989 free_extent_buffer(next);
Tsutomu Itoh97d9a8a2011-03-24 06:33:21 +00008990 return -EIO;
Josef Bacik416bc652013-04-23 14:17:42 -04008991 }
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008992 btrfs_tree_lock(next);
David Sterba8bead252018-04-04 02:03:48 +02008993 btrfs_set_lock_blocking_write(next);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04008994 }
8995
8996 level--;
Josef Bacik48672682016-09-23 13:23:28 +02008997 ASSERT(level == btrfs_header_level(next));
8998 if (level != btrfs_header_level(next)) {
8999 btrfs_err(root->fs_info, "mismatched level");
9000 ret = -EIO;
9001 goto out_unlock;
9002 }
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009003 path->nodes[level] = next;
9004 path->slots[level] = 0;
Chris Masonbd681512011-07-16 15:23:14 -04009005 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009006 wc->level = level;
9007 if (wc->level == 1)
9008 wc->reada_slot = 0;
9009 return 0;
9010skip:
9011 wc->refs[level - 1] = 0;
9012 wc->flags[level - 1] = 0;
Yan, Zheng94fcca92009-10-09 09:25:16 -04009013 if (wc->stage == DROP_REFERENCE) {
9014 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
9015 parent = path->nodes[level]->start;
9016 } else {
Josef Bacik48672682016-09-23 13:23:28 +02009017 ASSERT(root->root_key.objectid ==
Yan, Zheng94fcca92009-10-09 09:25:16 -04009018 btrfs_header_owner(path->nodes[level]));
Josef Bacik48672682016-09-23 13:23:28 +02009019 if (root->root_key.objectid !=
9020 btrfs_header_owner(path->nodes[level])) {
9021 btrfs_err(root->fs_info,
9022 "mismatched block owner");
9023 ret = -EIO;
9024 goto out_unlock;
9025 }
Yan, Zheng94fcca92009-10-09 09:25:16 -04009026 parent = 0;
9027 }
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009028
Qu Wenruo2cd86d32018-09-27 14:42:33 +08009029 /*
Josef Bacik78c52d92019-02-06 15:46:14 -05009030 * If we had a drop_progress we need to verify the refs are set
9031 * as expected. If we find our ref then we know that from here
9032 * on out everything should be correct, and we can clear the
9033 * ->restarted flag.
9034 */
9035 if (wc->restarted) {
9036 ret = check_ref_exists(trans, root, bytenr, parent,
9037 level - 1);
9038 if (ret < 0)
9039 goto out_unlock;
9040 if (ret == 0)
9041 goto no_delete;
9042 ret = 0;
9043 wc->restarted = 0;
9044 }
9045
9046 /*
Qu Wenruo2cd86d32018-09-27 14:42:33 +08009047 * Reloc tree doesn't contribute to qgroup numbers, and we have
9048 * already accounted them at merge time (replace_path),
9049 * thus we could skip expensive subtree trace here.
9050 */
9051 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
9052 need_account) {
Lu Fengqideb40622018-07-18 14:45:38 +08009053 ret = btrfs_qgroup_trace_subtree(trans, next,
Qu Wenruo33d1f052016-10-18 09:31:28 +08009054 generation, level - 1);
Mark Fasheh11526512014-07-17 12:39:01 -07009055 if (ret) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009056 btrfs_err_rl(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04009057 "Error %d accounting shared subtree. Quota is out of sync, rescan required.",
9058 ret);
Mark Fasheh11526512014-07-17 12:39:01 -07009059 }
9060 }
Josef Bacikaea6f022019-02-06 15:46:15 -05009061
9062 /*
9063 * We need to update the next key in our walk control so we can
9064 * update the drop_progress key accordingly. We don't care if
9065 * find_next_key doesn't find a key because that means we're at
9066 * the end and are going to clean up now.
9067 */
9068 wc->drop_level = level;
9069 find_next_key(path, level, &wc->drop_progress);
9070
Qu Wenruo01e0da42018-12-10 15:01:03 +08009071 ret = btrfs_free_extent(trans, root, bytenr, fs_info->nodesize,
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009072 parent, root->root_key.objectid,
9073 level - 1, 0);
Josef Bacik48672682016-09-23 13:23:28 +02009074 if (ret)
9075 goto out_unlock;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009076 }
Josef Bacik78c52d92019-02-06 15:46:14 -05009077no_delete:
Josef Bacik48672682016-09-23 13:23:28 +02009078 *lookup_info = 1;
9079 ret = 1;
9080
9081out_unlock:
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009082 btrfs_tree_unlock(next);
9083 free_extent_buffer(next);
Josef Bacik48672682016-09-23 13:23:28 +02009084
9085 return ret;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009086}
9087
9088/*
Liu Bo2c016dc2012-12-26 15:32:17 +08009089 * helper to process tree block while walking up the tree.
Yan Zheng2c47e6052009-06-27 21:07:35 -04009090 *
9091 * when wc->stage == DROP_REFERENCE, this function drops
9092 * reference count on the block.
9093 *
9094 * when wc->stage == UPDATE_BACKREF, this function changes
9095 * wc->stage back to DROP_REFERENCE if we changed wc->stage
9096 * to UPDATE_BACKREF previously while processing the block.
9097 *
9098 * NOTE: return value 1 means we should stop walking up.
9099 */
9100static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
9101 struct btrfs_root *root,
9102 struct btrfs_path *path,
9103 struct walk_control *wc)
9104{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009105 struct btrfs_fs_info *fs_info = root->fs_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04009106 int ret;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009107 int level = wc->level;
9108 struct extent_buffer *eb = path->nodes[level];
9109 u64 parent = 0;
9110
9111 if (wc->stage == UPDATE_BACKREF) {
9112 BUG_ON(wc->shared_level < level);
9113 if (level < wc->shared_level)
9114 goto out;
9115
Yan Zheng2c47e6052009-06-27 21:07:35 -04009116 ret = find_next_key(path, level + 1, &wc->update_progress);
9117 if (ret > 0)
9118 wc->update_ref = 0;
9119
9120 wc->stage = DROP_REFERENCE;
9121 wc->shared_level = -1;
9122 path->slots[level] = 0;
9123
9124 /*
9125 * check reference count again if the block isn't locked.
9126 * we should start walking down the tree again if reference
9127 * count is one.
9128 */
9129 if (!path->locks[level]) {
9130 BUG_ON(level == 0);
9131 btrfs_tree_lock(eb);
David Sterba8bead252018-04-04 02:03:48 +02009132 btrfs_set_lock_blocking_write(eb);
Chris Masonbd681512011-07-16 15:23:14 -04009133 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009134
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009135 ret = btrfs_lookup_extent_info(trans, fs_info,
Josef Bacik3173a182013-03-07 14:22:04 -05009136 eb->start, level, 1,
Yan Zheng2c47e6052009-06-27 21:07:35 -04009137 &wc->refs[level],
9138 &wc->flags[level]);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009139 if (ret < 0) {
9140 btrfs_tree_unlock_rw(eb, path->locks[level]);
Liu Bo3268a242012-12-28 09:33:19 +00009141 path->locks[level] = 0;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009142 return ret;
9143 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009144 BUG_ON(wc->refs[level] == 0);
9145 if (wc->refs[level] == 1) {
Chris Masonbd681512011-07-16 15:23:14 -04009146 btrfs_tree_unlock_rw(eb, path->locks[level]);
Liu Bo3268a242012-12-28 09:33:19 +00009147 path->locks[level] = 0;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009148 return 1;
9149 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009150 }
9151 }
9152
9153 /* wc->stage == DROP_REFERENCE */
9154 BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
9155
9156 if (wc->refs[level] == 1) {
9157 if (level == 0) {
9158 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
Josef Bacike339a6b2014-07-02 10:54:25 -07009159 ret = btrfs_dec_ref(trans, root, eb, 1);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009160 else
Josef Bacike339a6b2014-07-02 10:54:25 -07009161 ret = btrfs_dec_ref(trans, root, eb, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009162 BUG_ON(ret); /* -ENOMEM */
Lu Fengqi8d38d7e2018-07-18 14:45:37 +08009163 ret = btrfs_qgroup_trace_leaf_items(trans, eb);
Mark Fasheh11526512014-07-17 12:39:01 -07009164 if (ret) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009165 btrfs_err_rl(fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04009166 "error %d accounting leaf items. Quota is out of sync, rescan required.",
9167 ret);
Mark Fasheh11526512014-07-17 12:39:01 -07009168 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009169 }
David Sterba6a884d7d2019-03-20 14:30:02 +01009170 /* make block locked assertion in btrfs_clean_tree_block happy */
Yan Zheng2c47e6052009-06-27 21:07:35 -04009171 if (!path->locks[level] &&
9172 btrfs_header_generation(eb) == trans->transid) {
9173 btrfs_tree_lock(eb);
David Sterba8bead252018-04-04 02:03:48 +02009174 btrfs_set_lock_blocking_write(eb);
Chris Masonbd681512011-07-16 15:23:14 -04009175 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009176 }
David Sterba6a884d7d2019-03-20 14:30:02 +01009177 btrfs_clean_tree_block(eb);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009178 }
9179
9180 if (eb == root->node) {
9181 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
9182 parent = eb->start;
Qu Wenruo65c6e822018-08-21 09:42:03 +08009183 else if (root->root_key.objectid != btrfs_header_owner(eb))
9184 goto owner_mismatch;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009185 } else {
9186 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
9187 parent = path->nodes[level + 1]->start;
Qu Wenruo65c6e822018-08-21 09:42:03 +08009188 else if (root->root_key.objectid !=
9189 btrfs_header_owner(path->nodes[level + 1]))
9190 goto owner_mismatch;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009191 }
9192
Jan Schmidt5581a512012-05-16 17:04:52 +02009193 btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009194out:
9195 wc->refs[level] = 0;
9196 wc->flags[level] = 0;
Yan, Zhengf0486c62010-05-16 10:46:25 -04009197 return 0;
Qu Wenruo65c6e822018-08-21 09:42:03 +08009198
9199owner_mismatch:
9200 btrfs_err_rl(fs_info, "unexpected tree owner, have %llu expect %llu",
9201 btrfs_header_owner(eb), root->root_key.objectid);
9202 return -EUCLEAN;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009203}
9204
Yan Zheng5d4f98a2009-06-10 10:45:14 -04009205static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
9206 struct btrfs_root *root,
Yan Zheng2c47e6052009-06-27 21:07:35 -04009207 struct btrfs_path *path,
9208 struct walk_control *wc)
Yan Zhengf82d02d2008-10-29 14:49:05 -04009209{
Yan Zheng2c47e6052009-06-27 21:07:35 -04009210 int level = wc->level;
Yan, Zheng94fcca92009-10-09 09:25:16 -04009211 int lookup_info = 1;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009212 int ret;
9213
Yan Zheng2c47e6052009-06-27 21:07:35 -04009214 while (level >= 0) {
Yan, Zheng94fcca92009-10-09 09:25:16 -04009215 ret = walk_down_proc(trans, root, path, wc, lookup_info);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009216 if (ret > 0)
Yan Zhengf82d02d2008-10-29 14:49:05 -04009217 break;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009218
Yan Zheng2c47e6052009-06-27 21:07:35 -04009219 if (level == 0)
9220 break;
9221
Yan, Zheng7a7965f2010-02-01 02:41:17 +00009222 if (path->slots[level] >=
9223 btrfs_header_nritems(path->nodes[level]))
9224 break;
9225
Yan, Zheng94fcca92009-10-09 09:25:16 -04009226 ret = do_walk_down(trans, root, path, wc, &lookup_info);
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009227 if (ret > 0) {
9228 path->slots[level]++;
9229 continue;
Miao Xie90d2c51d2010-03-25 12:37:12 +00009230 } else if (ret < 0)
9231 return ret;
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009232 level = wc->level;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009233 }
Yan Zhengf82d02d2008-10-29 14:49:05 -04009234 return 0;
9235}
9236
Chris Masond3977122009-01-05 21:25:51 -05009237static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05009238 struct btrfs_root *root,
Yan Zhengf82d02d2008-10-29 14:49:05 -04009239 struct btrfs_path *path,
Yan Zheng2c47e6052009-06-27 21:07:35 -04009240 struct walk_control *wc, int max_level)
Chris Mason20524f02007-03-10 06:35:47 -05009241{
Yan Zheng2c47e6052009-06-27 21:07:35 -04009242 int level = wc->level;
Chris Mason20524f02007-03-10 06:35:47 -05009243 int ret;
Chris Mason9f3a7422007-08-07 15:52:19 -04009244
Yan Zheng2c47e6052009-06-27 21:07:35 -04009245 path->slots[level] = btrfs_header_nritems(path->nodes[level]);
9246 while (level < max_level && path->nodes[level]) {
9247 wc->level = level;
9248 if (path->slots[level] + 1 <
9249 btrfs_header_nritems(path->nodes[level])) {
9250 path->slots[level]++;
Chris Mason20524f02007-03-10 06:35:47 -05009251 return 0;
9252 } else {
Yan Zheng2c47e6052009-06-27 21:07:35 -04009253 ret = walk_up_proc(trans, root, path, wc);
9254 if (ret > 0)
9255 return 0;
Qu Wenruo65c6e822018-08-21 09:42:03 +08009256 if (ret < 0)
9257 return ret;
Chris Masonbd56b302009-02-04 09:27:02 -05009258
Yan Zheng2c47e6052009-06-27 21:07:35 -04009259 if (path->locks[level]) {
Chris Masonbd681512011-07-16 15:23:14 -04009260 btrfs_tree_unlock_rw(path->nodes[level],
9261 path->locks[level]);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009262 path->locks[level] = 0;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009263 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009264 free_extent_buffer(path->nodes[level]);
9265 path->nodes[level] = NULL;
9266 level++;
Chris Mason20524f02007-03-10 06:35:47 -05009267 }
9268 }
9269 return 1;
9270}
9271
Chris Mason9aca1d52007-03-13 11:09:37 -04009272/*
Yan Zheng2c47e6052009-06-27 21:07:35 -04009273 * drop a subvolume tree.
9274 *
9275 * this function traverses the tree freeing any blocks that only
9276 * referenced by the tree.
9277 *
9278 * when a shared tree block is found. this function decreases its
9279 * reference count by one. if update_ref is true, this function
9280 * also make sure backrefs for the shared block and all lower level
9281 * blocks are properly updated.
David Sterba9d1a2a32013-03-12 15:13:28 +00009282 *
9283 * If called with for_reloc == 0, may exit early with -EAGAIN
Chris Mason9aca1d52007-03-13 11:09:37 -04009284 */
Jeff Mahoney2c536792011-10-03 23:22:41 -04009285int btrfs_drop_snapshot(struct btrfs_root *root,
Arne Jansen66d7e7f2011-09-12 15:26:38 +02009286 struct btrfs_block_rsv *block_rsv, int update_ref,
9287 int for_reloc)
Chris Mason20524f02007-03-10 06:35:47 -05009288{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04009289 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason5caf2a02007-04-02 11:20:42 -04009290 struct btrfs_path *path;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009291 struct btrfs_trans_handle *trans;
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04009292 struct btrfs_root *tree_root = fs_info->tree_root;
Chris Mason9f3a7422007-08-07 15:52:19 -04009293 struct btrfs_root_item *root_item = &root->root_item;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009294 struct walk_control *wc;
9295 struct btrfs_key key;
9296 int err = 0;
9297 int ret;
9298 int level;
Josef Bacikd29a9f62013-07-17 19:30:20 -04009299 bool root_dropped = false;
Chris Mason20524f02007-03-10 06:35:47 -05009300
Misono Tomohiro4fd786e2018-08-06 14:25:24 +09009301 btrfs_debug(fs_info, "Drop subvolume %llu", root->root_key.objectid);
Mark Fasheh11526512014-07-17 12:39:01 -07009302
Chris Mason5caf2a02007-04-02 11:20:42 -04009303 path = btrfs_alloc_path();
Tsutomu Itohcb1b69f2011-08-09 07:11:13 +00009304 if (!path) {
9305 err = -ENOMEM;
9306 goto out;
9307 }
Chris Mason20524f02007-03-10 06:35:47 -05009308
Yan Zheng2c47e6052009-06-27 21:07:35 -04009309 wc = kzalloc(sizeof(*wc), GFP_NOFS);
Mark Fasheh38a1a912011-07-13 10:59:59 -07009310 if (!wc) {
9311 btrfs_free_path(path);
Tsutomu Itohcb1b69f2011-08-09 07:11:13 +00009312 err = -ENOMEM;
9313 goto out;
Mark Fasheh38a1a912011-07-13 10:59:59 -07009314 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009315
Yan, Zhenga22285a2010-05-16 10:48:46 -04009316 trans = btrfs_start_transaction(tree_root, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009317 if (IS_ERR(trans)) {
9318 err = PTR_ERR(trans);
9319 goto out_free;
9320 }
Tsutomu Itoh98d5dc12011-01-20 06:19:37 +00009321
Josef Bacik0568e822018-11-30 11:52:14 -05009322 err = btrfs_run_delayed_items(trans);
9323 if (err)
9324 goto out_end_trans;
9325
Yan, Zheng3fd0a552010-05-16 10:49:59 -04009326 if (block_rsv)
9327 trans->block_rsv = block_rsv;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009328
Josef Bacik83354f02018-11-30 11:52:13 -05009329 /*
9330 * This will help us catch people modifying the fs tree while we're
9331 * dropping it. It is unsafe to mess with the fs tree while it's being
9332 * dropped as we unlock the root node and parent nodes as we walk down
9333 * the tree, assuming nothing will change. If something does change
9334 * then we'll have stale information and drop references to blocks we've
9335 * already dropped.
9336 */
9337 set_bit(BTRFS_ROOT_DELETING, &root->state);
Chris Mason9f3a7422007-08-07 15:52:19 -04009338 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
Yan Zheng2c47e6052009-06-27 21:07:35 -04009339 level = btrfs_header_level(root->node);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04009340 path->nodes[level] = btrfs_lock_root_node(root);
David Sterba8bead252018-04-04 02:03:48 +02009341 btrfs_set_lock_blocking_write(path->nodes[level]);
Chris Mason9f3a7422007-08-07 15:52:19 -04009342 path->slots[level] = 0;
Chris Masonbd681512011-07-16 15:23:14 -04009343 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009344 memset(&wc->update_progress, 0,
9345 sizeof(wc->update_progress));
Chris Mason9f3a7422007-08-07 15:52:19 -04009346 } else {
Chris Mason9f3a7422007-08-07 15:52:19 -04009347 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009348 memcpy(&wc->update_progress, &key,
9349 sizeof(wc->update_progress));
9350
Chris Mason6702ed42007-08-07 16:15:09 -04009351 level = root_item->drop_level;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009352 BUG_ON(level == 0);
Chris Mason6702ed42007-08-07 16:15:09 -04009353 path->lowest_level = level;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009354 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
9355 path->lowest_level = 0;
9356 if (ret < 0) {
9357 err = ret;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009358 goto out_end_trans;
Chris Mason9f3a7422007-08-07 15:52:19 -04009359 }
Yan, Zheng1c4850e2009-09-21 15:55:59 -04009360 WARN_ON(ret > 0);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009361
Chris Mason7d9eb122008-07-08 14:19:17 -04009362 /*
9363 * unlock our path, this is safe because only this
9364 * function is allowed to delete this snapshot
9365 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04009366 btrfs_unlock_up_safe(path, 0);
Chris Mason9aca1d52007-03-13 11:09:37 -04009367
Yan Zheng2c47e6052009-06-27 21:07:35 -04009368 level = btrfs_header_level(root->node);
9369 while (1) {
9370 btrfs_tree_lock(path->nodes[level]);
David Sterba8bead252018-04-04 02:03:48 +02009371 btrfs_set_lock_blocking_write(path->nodes[level]);
Josef Bacikfec386a2013-07-15 12:41:42 -04009372 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009373
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009374 ret = btrfs_lookup_extent_info(trans, fs_info,
Yan Zheng2c47e6052009-06-27 21:07:35 -04009375 path->nodes[level]->start,
Josef Bacik3173a182013-03-07 14:22:04 -05009376 level, 1, &wc->refs[level],
Yan Zheng2c47e6052009-06-27 21:07:35 -04009377 &wc->flags[level]);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009378 if (ret < 0) {
9379 err = ret;
9380 goto out_end_trans;
9381 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009382 BUG_ON(wc->refs[level] == 0);
9383
9384 if (level == root_item->drop_level)
9385 break;
9386
9387 btrfs_tree_unlock(path->nodes[level]);
Josef Bacikfec386a2013-07-15 12:41:42 -04009388 path->locks[level] = 0;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009389 WARN_ON(wc->refs[level] != 1);
9390 level--;
9391 }
9392 }
9393
Josef Bacik78c52d92019-02-06 15:46:14 -05009394 wc->restarted = test_bit(BTRFS_ROOT_DEAD_TREE, &root->state);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009395 wc->level = level;
9396 wc->shared_level = -1;
9397 wc->stage = DROP_REFERENCE;
9398 wc->update_ref = update_ref;
9399 wc->keep_locks = 0;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009400 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009401
9402 while (1) {
David Sterba9d1a2a32013-03-12 15:13:28 +00009403
Yan Zheng2c47e6052009-06-27 21:07:35 -04009404 ret = walk_down_tree(trans, root, path, wc);
9405 if (ret < 0) {
9406 err = ret;
Chris Masone7a84562008-06-25 16:01:31 -04009407 break;
9408 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009409
9410 ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
9411 if (ret < 0) {
9412 err = ret;
9413 break;
9414 }
9415
9416 if (ret > 0) {
9417 BUG_ON(wc->stage != DROP_REFERENCE);
9418 break;
9419 }
9420
9421 if (wc->stage == DROP_REFERENCE) {
Josef Bacikaea6f022019-02-06 15:46:15 -05009422 wc->drop_level = wc->level;
9423 btrfs_node_key_to_cpu(path->nodes[wc->drop_level],
9424 &wc->drop_progress,
9425 path->slots[wc->drop_level]);
Yan Zheng2c47e6052009-06-27 21:07:35 -04009426 }
Josef Bacikaea6f022019-02-06 15:46:15 -05009427 btrfs_cpu_key_to_disk(&root_item->drop_progress,
9428 &wc->drop_progress);
9429 root_item->drop_level = wc->drop_level;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009430
9431 BUG_ON(wc->level == 0);
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04009432 if (btrfs_should_end_transaction(trans) ||
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009433 (!for_reloc && btrfs_need_cleaner_sleep(fs_info))) {
Yan Zheng2c47e6052009-06-27 21:07:35 -04009434 ret = btrfs_update_root(trans, tree_root,
9435 &root->root_key,
9436 root_item);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009437 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04009438 btrfs_abort_transaction(trans, ret);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009439 err = ret;
9440 goto out_end_trans;
9441 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009442
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04009443 btrfs_end_transaction_throttle(trans);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009444 if (!for_reloc && btrfs_need_cleaner_sleep(fs_info)) {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04009445 btrfs_debug(fs_info,
9446 "drop snapshot early exit");
Josef Bacik3c8f2422013-07-15 11:57:06 -04009447 err = -EAGAIN;
9448 goto out_free;
9449 }
9450
Yan, Zhenga22285a2010-05-16 10:48:46 -04009451 trans = btrfs_start_transaction(tree_root, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009452 if (IS_ERR(trans)) {
9453 err = PTR_ERR(trans);
9454 goto out_free;
9455 }
Yan, Zheng3fd0a552010-05-16 10:49:59 -04009456 if (block_rsv)
9457 trans->block_rsv = block_rsv;
Chris Masonc3e69d52009-03-13 10:17:05 -04009458 }
Chris Mason20524f02007-03-10 06:35:47 -05009459 }
David Sterbab3b4aa72011-04-21 01:20:15 +02009460 btrfs_release_path(path);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009461 if (err)
9462 goto out_end_trans;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009463
Lu Fengqiab9ce7d2018-08-01 11:32:27 +08009464 ret = btrfs_del_root(trans, &root->root_key);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009465 if (ret) {
Jeff Mahoney66642832016-06-10 18:19:25 -04009466 btrfs_abort_transaction(trans, ret);
Jeff Mahoneye19182c2017-12-04 13:11:45 -05009467 err = ret;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009468 goto out_end_trans;
9469 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009470
Yan, Zheng76dda932009-09-21 16:00:26 -04009471 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
Miao Xiecb517ea2013-05-15 07:48:19 +00009472 ret = btrfs_find_root(tree_root, &root->root_key, path,
9473 NULL, NULL);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009474 if (ret < 0) {
Jeff Mahoney66642832016-06-10 18:19:25 -04009475 btrfs_abort_transaction(trans, ret);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009476 err = ret;
9477 goto out_end_trans;
9478 } else if (ret > 0) {
Josef Bacik84cd9482010-12-08 12:24:01 -05009479 /* if we fail to delete the orphan item this time
9480 * around, it'll get picked up the next time.
9481 *
9482 * The most common failure here is just -ENOENT.
9483 */
9484 btrfs_del_orphan_item(trans, tree_root,
9485 root->root_key.objectid);
Yan, Zheng76dda932009-09-21 16:00:26 -04009486 }
9487 }
9488
Miao Xie27cdeb72014-04-02 19:51:05 +08009489 if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) {
Josef Bacik2b9dbef2015-09-15 10:07:04 -04009490 btrfs_add_dropped_root(trans, root);
Yan, Zheng76dda932009-09-21 16:00:26 -04009491 } else {
9492 free_extent_buffer(root->node);
9493 free_extent_buffer(root->commit_root);
Miao Xieb0feb9d2013-05-15 07:48:20 +00009494 btrfs_put_fs_root(root);
Yan, Zheng76dda932009-09-21 16:00:26 -04009495 }
Josef Bacikd29a9f62013-07-17 19:30:20 -04009496 root_dropped = true;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009497out_end_trans:
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04009498 btrfs_end_transaction_throttle(trans);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009499out_free:
Yan Zheng2c47e6052009-06-27 21:07:35 -04009500 kfree(wc);
Chris Mason5caf2a02007-04-02 11:20:42 -04009501 btrfs_free_path(path);
Tsutomu Itohcb1b69f2011-08-09 07:11:13 +00009502out:
Josef Bacikd29a9f62013-07-17 19:30:20 -04009503 /*
9504 * So if we need to stop dropping the snapshot for whatever reason we
9505 * need to make sure to add it back to the dead root list so that we
9506 * keep trying to do the work later. This also cleans up roots if we
9507 * don't have it in the radix (like when we recover after a power fail
9508 * or unmount) so we don't leak memory.
9509 */
Thomas Meyer897ca812017-10-07 16:02:21 +02009510 if (!for_reloc && !root_dropped)
Josef Bacikd29a9f62013-07-17 19:30:20 -04009511 btrfs_add_dead_root(root);
Wang Shilong90515e72014-01-07 17:26:58 +08009512 if (err && err != -EAGAIN)
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04009513 btrfs_handle_fs_error(fs_info, err, NULL);
Jeff Mahoney2c536792011-10-03 23:22:41 -04009514 return err;
Chris Mason20524f02007-03-10 06:35:47 -05009515}
Chris Mason9078a3e2007-04-26 16:46:15 -04009516
Yan Zheng2c47e6052009-06-27 21:07:35 -04009517/*
9518 * drop subtree rooted at tree block 'node'.
9519 *
9520 * NOTE: this function will unlock and release tree block 'node'
Arne Jansen66d7e7f2011-09-12 15:26:38 +02009521 * only used by relocation code
Yan Zheng2c47e6052009-06-27 21:07:35 -04009522 */
Yan Zhengf82d02d2008-10-29 14:49:05 -04009523int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
9524 struct btrfs_root *root,
9525 struct extent_buffer *node,
9526 struct extent_buffer *parent)
9527{
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009528 struct btrfs_fs_info *fs_info = root->fs_info;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009529 struct btrfs_path *path;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009530 struct walk_control *wc;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009531 int level;
9532 int parent_level;
9533 int ret = 0;
9534 int wret;
9535
Yan Zheng2c47e6052009-06-27 21:07:35 -04009536 BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
9537
Yan Zhengf82d02d2008-10-29 14:49:05 -04009538 path = btrfs_alloc_path();
Tsutomu Itohdb5b4932011-03-23 08:14:16 +00009539 if (!path)
9540 return -ENOMEM;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009541
Yan Zheng2c47e6052009-06-27 21:07:35 -04009542 wc = kzalloc(sizeof(*wc), GFP_NOFS);
Tsutomu Itohdb5b4932011-03-23 08:14:16 +00009543 if (!wc) {
9544 btrfs_free_path(path);
9545 return -ENOMEM;
9546 }
Yan Zheng2c47e6052009-06-27 21:07:35 -04009547
Chris Masonb9447ef82009-03-09 11:45:38 -04009548 btrfs_assert_tree_locked(parent);
Yan Zhengf82d02d2008-10-29 14:49:05 -04009549 parent_level = btrfs_header_level(parent);
9550 extent_buffer_get(parent);
9551 path->nodes[parent_level] = parent;
9552 path->slots[parent_level] = btrfs_header_nritems(parent);
9553
Chris Masonb9447ef82009-03-09 11:45:38 -04009554 btrfs_assert_tree_locked(node);
Yan Zhengf82d02d2008-10-29 14:49:05 -04009555 level = btrfs_header_level(node);
Yan Zhengf82d02d2008-10-29 14:49:05 -04009556 path->nodes[level] = node;
9557 path->slots[level] = 0;
Chris Masonbd681512011-07-16 15:23:14 -04009558 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009559
9560 wc->refs[parent_level] = 1;
9561 wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
9562 wc->level = level;
9563 wc->shared_level = -1;
9564 wc->stage = DROP_REFERENCE;
9565 wc->update_ref = 0;
9566 wc->keep_locks = 1;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009567 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info);
Yan Zhengf82d02d2008-10-29 14:49:05 -04009568
9569 while (1) {
Yan Zheng2c47e6052009-06-27 21:07:35 -04009570 wret = walk_down_tree(trans, root, path, wc);
9571 if (wret < 0) {
Yan Zhengf82d02d2008-10-29 14:49:05 -04009572 ret = wret;
Yan Zhengf82d02d2008-10-29 14:49:05 -04009573 break;
Yan Zheng2c47e6052009-06-27 21:07:35 -04009574 }
Yan Zhengf82d02d2008-10-29 14:49:05 -04009575
Yan Zheng2c47e6052009-06-27 21:07:35 -04009576 wret = walk_up_tree(trans, root, path, wc, parent_level);
Yan Zhengf82d02d2008-10-29 14:49:05 -04009577 if (wret < 0)
9578 ret = wret;
9579 if (wret != 0)
9580 break;
9581 }
9582
Yan Zheng2c47e6052009-06-27 21:07:35 -04009583 kfree(wc);
Yan Zhengf82d02d2008-10-29 14:49:05 -04009584 btrfs_free_path(path);
9585 return ret;
9586}
9587
Jeff Mahoney6202df62016-06-22 18:54:22 -04009588static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags)
Chris Masonec44a352008-04-28 15:29:52 -04009589{
9590 u64 num_devices;
Ilya Dryomovfc67c452012-03-27 17:09:17 +03009591 u64 stripped;
Chris Masonec44a352008-04-28 15:29:52 -04009592
Ilya Dryomovfc67c452012-03-27 17:09:17 +03009593 /*
9594 * if restripe for this chunk_type is on pick target profile and
9595 * return, otherwise do the usual balance
9596 */
Jeff Mahoney6202df62016-06-22 18:54:22 -04009597 stripped = get_restripe_target(fs_info, flags);
Ilya Dryomovfc67c452012-03-27 17:09:17 +03009598 if (stripped)
9599 return extended_to_chunk(stripped);
Ilya Dryomove4d8ec02012-01-16 22:04:48 +02009600
Jeff Mahoney6202df62016-06-22 18:54:22 -04009601 num_devices = fs_info->fs_devices->rw_devices;
Chris Masoncd02dca2010-12-13 14:56:23 -05009602
Ilya Dryomovfc67c452012-03-27 17:09:17 +03009603 stripped = BTRFS_BLOCK_GROUP_RAID0 |
David Woodhouse53b381b2013-01-29 18:40:14 -05009604 BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 |
Ilya Dryomovfc67c452012-03-27 17:09:17 +03009605 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;
9606
Chris Masonec44a352008-04-28 15:29:52 -04009607 if (num_devices == 1) {
9608 stripped |= BTRFS_BLOCK_GROUP_DUP;
9609 stripped = flags & ~stripped;
9610
9611 /* turn raid0 into single device chunks */
9612 if (flags & BTRFS_BLOCK_GROUP_RAID0)
9613 return stripped;
9614
9615 /* turn mirroring into duplication */
9616 if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
9617 BTRFS_BLOCK_GROUP_RAID10))
9618 return stripped | BTRFS_BLOCK_GROUP_DUP;
Chris Masonec44a352008-04-28 15:29:52 -04009619 } else {
9620 /* they already had raid on here, just return */
Chris Masonec44a352008-04-28 15:29:52 -04009621 if (flags & stripped)
9622 return flags;
9623
9624 stripped |= BTRFS_BLOCK_GROUP_DUP;
9625 stripped = flags & ~stripped;
9626
9627 /* switch duplicated blocks with raid1 */
9628 if (flags & BTRFS_BLOCK_GROUP_DUP)
9629 return stripped | BTRFS_BLOCK_GROUP_RAID1;
9630
Ilya Dryomove3176ca2012-03-27 17:09:16 +03009631 /* this is drive concat, leave it alone */
Chris Masonec44a352008-04-28 15:29:52 -04009632 }
Ilya Dryomove3176ca2012-03-27 17:09:16 +03009633
Chris Masonec44a352008-04-28 15:29:52 -04009634 return flags;
9635}
9636
Zhaolei868f4012015-08-05 16:43:27 +08009637static int inc_block_group_ro(struct btrfs_block_group_cache *cache, int force)
Chris Mason0ef3e662008-05-24 14:04:53 -04009638{
Yan, Zhengf0486c62010-05-16 10:46:25 -04009639 struct btrfs_space_info *sinfo = cache->space_info;
9640 u64 num_bytes;
Qu Wenruo3ece54e2019-01-30 13:07:51 +08009641 u64 sinfo_used;
Miao Xie199c36e2011-07-15 10:34:36 +00009642 u64 min_allocable_bytes;
Yan, Zhengf0486c62010-05-16 10:46:25 -04009643 int ret = -ENOSPC;
Chris Mason0ef3e662008-05-24 14:04:53 -04009644
Miao Xie199c36e2011-07-15 10:34:36 +00009645 /*
9646 * We need some metadata space and system metadata space for
9647 * allocating chunks in some corner cases until we force to set
9648 * it to be readonly.
9649 */
9650 if ((sinfo->flags &
9651 (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
9652 !force)
Byongho Leeee221842015-12-15 01:42:10 +09009653 min_allocable_bytes = SZ_1M;
Miao Xie199c36e2011-07-15 10:34:36 +00009654 else
9655 min_allocable_bytes = 0;
9656
Yan, Zhengf0486c62010-05-16 10:46:25 -04009657 spin_lock(&sinfo->lock);
9658 spin_lock(&cache->lock);
WuBo61cfea92011-07-26 03:30:11 +00009659
9660 if (cache->ro) {
Zhaolei868f4012015-08-05 16:43:27 +08009661 cache->ro++;
WuBo61cfea92011-07-26 03:30:11 +00009662 ret = 0;
9663 goto out;
9664 }
9665
Yan, Zhengf0486c62010-05-16 10:46:25 -04009666 num_bytes = cache->key.offset - cache->reserved - cache->pinned -
9667 cache->bytes_super - btrfs_block_group_used(&cache->item);
Qu Wenruo3ece54e2019-01-30 13:07:51 +08009668 sinfo_used = btrfs_space_info_used(sinfo, true);
Chris Mason7d9eb122008-07-08 14:19:17 -04009669
Qu Wenruo3ece54e2019-01-30 13:07:51 +08009670 if (sinfo_used + num_bytes + min_allocable_bytes <=
9671 sinfo->total_bytes) {
Yan, Zhengf0486c62010-05-16 10:46:25 -04009672 sinfo->bytes_readonly += num_bytes;
Zhaolei868f4012015-08-05 16:43:27 +08009673 cache->ro++;
Josef Bacik633c0aa2014-10-31 09:49:34 -04009674 list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
Yan, Zhengf0486c62010-05-16 10:46:25 -04009675 ret = 0;
9676 }
WuBo61cfea92011-07-26 03:30:11 +00009677out:
Yan, Zhengf0486c62010-05-16 10:46:25 -04009678 spin_unlock(&cache->lock);
9679 spin_unlock(&sinfo->lock);
Qu Wenruo3ece54e2019-01-30 13:07:51 +08009680 if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
9681 btrfs_info(cache->fs_info,
9682 "unable to make block group %llu ro",
9683 cache->key.objectid);
9684 btrfs_info(cache->fs_info,
9685 "sinfo_used=%llu bg_num_bytes=%llu min_allocable=%llu",
9686 sinfo_used, num_bytes, min_allocable_bytes);
9687 dump_space_info(cache->fs_info, cache->space_info, 0, 0);
9688 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04009689 return ret;
Chris Mason0ef3e662008-05-24 14:04:53 -04009690}
9691
Nikolay Borisovc83488a2018-06-20 15:49:14 +03009692int btrfs_inc_block_group_ro(struct btrfs_block_group_cache *cache)
Yan Zheng5d4f98a2009-06-10 10:45:14 -04009693
9694{
Nikolay Borisovc83488a2018-06-20 15:49:14 +03009695 struct btrfs_fs_info *fs_info = cache->fs_info;
Yan, Zhengf0486c62010-05-16 10:46:25 -04009696 struct btrfs_trans_handle *trans;
9697 u64 alloc_flags;
9698 int ret;
9699
Chris Mason1bbc6212015-04-06 12:46:08 -07009700again:
Jeff Mahoney5e00f192017-02-15 16:28:29 -05009701 trans = btrfs_join_transaction(fs_info->extent_root);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01009702 if (IS_ERR(trans))
9703 return PTR_ERR(trans);
Yan, Zhengf0486c62010-05-16 10:46:25 -04009704
Chris Mason1bbc6212015-04-06 12:46:08 -07009705 /*
9706 * we're not allowed to set block groups readonly after the dirty
9707 * block groups cache has started writing. If it already started,
9708 * back off and let this transaction commit
9709 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009710 mutex_lock(&fs_info->ro_block_group_mutex);
Josef Bacik3204d332015-09-24 10:46:10 -04009711 if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) {
Chris Mason1bbc6212015-04-06 12:46:08 -07009712 u64 transid = trans->transid;
9713
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009714 mutex_unlock(&fs_info->ro_block_group_mutex);
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04009715 btrfs_end_transaction(trans);
Chris Mason1bbc6212015-04-06 12:46:08 -07009716
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009717 ret = btrfs_wait_for_commit(fs_info, transid);
Chris Mason1bbc6212015-04-06 12:46:08 -07009718 if (ret)
9719 return ret;
9720 goto again;
9721 }
9722
Chris Mason153c35b2015-05-19 18:54:41 -07009723 /*
9724 * if we are changing raid levels, try to allocate a corresponding
9725 * block group with the new raid level.
9726 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009727 alloc_flags = update_block_group_flags(fs_info, cache->flags);
Chris Mason153c35b2015-05-19 18:54:41 -07009728 if (alloc_flags != cache->flags) {
Nikolay Borisov01458822018-06-20 15:49:05 +03009729 ret = do_chunk_alloc(trans, alloc_flags,
Chris Mason153c35b2015-05-19 18:54:41 -07009730 CHUNK_ALLOC_FORCE);
9731 /*
9732 * ENOSPC is allowed here, we may have enough space
9733 * already allocated at the new raid level to
9734 * carry on
9735 */
9736 if (ret == -ENOSPC)
9737 ret = 0;
9738 if (ret < 0)
9739 goto out;
9740 }
Chris Mason1bbc6212015-04-06 12:46:08 -07009741
Zhaolei868f4012015-08-05 16:43:27 +08009742 ret = inc_block_group_ro(cache, 0);
Yan, Zhengf0486c62010-05-16 10:46:25 -04009743 if (!ret)
9744 goto out;
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009745 alloc_flags = get_alloc_profile(fs_info, cache->space_info->flags);
Nikolay Borisov01458822018-06-20 15:49:05 +03009746 ret = do_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
Yan, Zhengf0486c62010-05-16 10:46:25 -04009747 if (ret < 0)
9748 goto out;
Zhaolei868f4012015-08-05 16:43:27 +08009749 ret = inc_block_group_ro(cache, 0);
Yan, Zhengf0486c62010-05-16 10:46:25 -04009750out:
Shaohua Li2f081082015-01-09 10:40:15 -08009751 if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009752 alloc_flags = update_block_group_flags(fs_info, cache->flags);
David Sterba34441362016-10-04 19:34:27 +02009753 mutex_lock(&fs_info->chunk_mutex);
Nikolay Borisov451a2c12018-06-20 15:49:07 +03009754 check_system_chunk(trans, alloc_flags);
David Sterba34441362016-10-04 19:34:27 +02009755 mutex_unlock(&fs_info->chunk_mutex);
Shaohua Li2f081082015-01-09 10:40:15 -08009756 }
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009757 mutex_unlock(&fs_info->ro_block_group_mutex);
Shaohua Li2f081082015-01-09 10:40:15 -08009758
Jeff Mahoney3a45bb22016-09-09 21:39:03 -04009759 btrfs_end_transaction(trans);
Yan, Zhengf0486c62010-05-16 10:46:25 -04009760 return ret;
9761}
9762
Nikolay Borisov43a7e992018-06-20 15:49:15 +03009763int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type)
Chris Masonc87f08c2011-02-16 13:57:04 -05009764{
Nikolay Borisov43a7e992018-06-20 15:49:15 +03009765 u64 alloc_flags = get_alloc_profile(trans->fs_info, type);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009766
Nikolay Borisov01458822018-06-20 15:49:05 +03009767 return do_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
Chris Masonc87f08c2011-02-16 13:57:04 -05009768}
9769
Miao Xie6d07bce2011-01-05 10:07:31 +00009770/*
9771 * helper to account the unused space of all the readonly block group in the
Josef Bacik633c0aa2014-10-31 09:49:34 -04009772 * space_info. takes mirrors into account.
Miao Xie6d07bce2011-01-05 10:07:31 +00009773 */
Josef Bacik633c0aa2014-10-31 09:49:34 -04009774u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
Miao Xie6d07bce2011-01-05 10:07:31 +00009775{
9776 struct btrfs_block_group_cache *block_group;
9777 u64 free_bytes = 0;
9778 int factor;
9779
Nicholas D Steeves01327612016-05-19 21:18:45 -04009780 /* It's df, we don't care if it's racy */
Josef Bacik633c0aa2014-10-31 09:49:34 -04009781 if (list_empty(&sinfo->ro_bgs))
9782 return 0;
9783
9784 spin_lock(&sinfo->lock);
9785 list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) {
Miao Xie6d07bce2011-01-05 10:07:31 +00009786 spin_lock(&block_group->lock);
9787
9788 if (!block_group->ro) {
9789 spin_unlock(&block_group->lock);
9790 continue;
9791 }
9792
David Sterba46df06b2018-07-13 20:46:30 +02009793 factor = btrfs_bg_type_to_factor(block_group->flags);
Miao Xie6d07bce2011-01-05 10:07:31 +00009794 free_bytes += (block_group->key.offset -
9795 btrfs_block_group_used(&block_group->item)) *
9796 factor;
9797
9798 spin_unlock(&block_group->lock);
9799 }
Miao Xie6d07bce2011-01-05 10:07:31 +00009800 spin_unlock(&sinfo->lock);
9801
9802 return free_bytes;
9803}
9804
Jeff Mahoney2ff7e612016-06-22 18:54:24 -04009805void btrfs_dec_block_group_ro(struct btrfs_block_group_cache *cache)
Yan, Zhengf0486c62010-05-16 10:46:25 -04009806{
9807 struct btrfs_space_info *sinfo = cache->space_info;
9808 u64 num_bytes;
9809
9810 BUG_ON(!cache->ro);
9811
9812 spin_lock(&sinfo->lock);
9813 spin_lock(&cache->lock);
Zhaolei868f4012015-08-05 16:43:27 +08009814 if (!--cache->ro) {
9815 num_bytes = cache->key.offset - cache->reserved -
9816 cache->pinned - cache->bytes_super -
9817 btrfs_block_group_used(&cache->item);
9818 sinfo->bytes_readonly -= num_bytes;
9819 list_del_init(&cache->ro_list);
9820 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04009821 spin_unlock(&cache->lock);
9822 spin_unlock(&sinfo->lock);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04009823}
9824
Josef Bacikba1bf482009-09-11 16:11:19 -04009825/*
Andrea Gelmini52042d82018-11-28 12:05:13 +01009826 * Checks to see if it's even possible to relocate this block group.
Josef Bacikba1bf482009-09-11 16:11:19 -04009827 *
9828 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
9829 * ok to go ahead and try.
9830 */
Jeff Mahoney6bccf3a2016-06-21 21:16:51 -04009831int btrfs_can_relocate(struct btrfs_fs_info *fs_info, u64 bytenr)
Zheng Yan1a40e232008-09-26 10:09:34 -04009832{
Zheng Yan1a40e232008-09-26 10:09:34 -04009833 struct btrfs_block_group_cache *block_group;
Josef Bacikba1bf482009-09-11 16:11:19 -04009834 struct btrfs_space_info *space_info;
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009835 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
Josef Bacikba1bf482009-09-11 16:11:19 -04009836 struct btrfs_device *device;
liubocdcb7252011-08-03 10:15:25 +00009837 u64 min_free;
Josef Bacik6719db62011-08-20 08:29:51 -04009838 u64 dev_min = 1;
9839 u64 dev_nr = 0;
Ilya Dryomov4a5e98f2012-03-27 17:09:17 +03009840 u64 target;
Qu Wenruo0305bc22016-03-23 11:38:17 +08009841 int debug;
liubocdcb7252011-08-03 10:15:25 +00009842 int index;
Josef Bacikba1bf482009-09-11 16:11:19 -04009843 int full = 0;
9844 int ret = 0;
Chris Masonedbd8d42007-12-21 16:27:24 -05009845
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009846 debug = btrfs_test_opt(fs_info, ENOSPC_DEBUG);
Qu Wenruo0305bc22016-03-23 11:38:17 +08009847
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009848 block_group = btrfs_lookup_block_group(fs_info, bytenr);
Zheng Yan1a40e232008-09-26 10:09:34 -04009849
Josef Bacikba1bf482009-09-11 16:11:19 -04009850 /* odd, couldn't find the block group, leave it alone */
Qu Wenruo0305bc22016-03-23 11:38:17 +08009851 if (!block_group) {
9852 if (debug)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009853 btrfs_warn(fs_info,
Qu Wenruo0305bc22016-03-23 11:38:17 +08009854 "can't find block group for bytenr %llu",
9855 bytenr);
Josef Bacikba1bf482009-09-11 16:11:19 -04009856 return -1;
Qu Wenruo0305bc22016-03-23 11:38:17 +08009857 }
Chris Masonedbd8d42007-12-21 16:27:24 -05009858
liubocdcb7252011-08-03 10:15:25 +00009859 min_free = btrfs_block_group_used(&block_group->item);
9860
Josef Bacikba1bf482009-09-11 16:11:19 -04009861 /* no bytes used, we're good */
liubocdcb7252011-08-03 10:15:25 +00009862 if (!min_free)
Josef Bacikba1bf482009-09-11 16:11:19 -04009863 goto out;
Chris Mason323da792008-05-09 11:46:48 -04009864
Josef Bacikba1bf482009-09-11 16:11:19 -04009865 space_info = block_group->space_info;
9866 spin_lock(&space_info->lock);
Chris Mason323da792008-05-09 11:46:48 -04009867
Josef Bacikba1bf482009-09-11 16:11:19 -04009868 full = space_info->full;
Zheng Yan1a40e232008-09-26 10:09:34 -04009869
Josef Bacikba1bf482009-09-11 16:11:19 -04009870 /*
9871 * if this is the last block group we have in this space, we can't
Chris Mason7ce618d2009-09-22 14:48:44 -04009872 * relocate it unless we're able to allocate a new chunk below.
9873 *
9874 * Otherwise, we need to make sure we have room in the space to handle
9875 * all of the extents from this block group. If we can, we're good
Josef Bacikba1bf482009-09-11 16:11:19 -04009876 */
Chris Mason7ce618d2009-09-22 14:48:44 -04009877 if ((space_info->total_bytes != block_group->key.offset) &&
Liu Bo41361352017-02-13 15:42:21 -08009878 (btrfs_space_info_used(space_info, false) + min_free <
9879 space_info->total_bytes)) {
Josef Bacikba1bf482009-09-11 16:11:19 -04009880 spin_unlock(&space_info->lock);
9881 goto out;
9882 }
9883 spin_unlock(&space_info->lock);
Zheng Yan1a40e232008-09-26 10:09:34 -04009884
Josef Bacikba1bf482009-09-11 16:11:19 -04009885 /*
9886 * ok we don't have enough space, but maybe we have free space on our
9887 * devices to allocate new chunks for relocation, so loop through our
Ilya Dryomov4a5e98f2012-03-27 17:09:17 +03009888 * alloc devices and guess if we have enough space. if this block
9889 * group is going to be restriped, run checks against the target
9890 * profile instead of the current one.
Josef Bacikba1bf482009-09-11 16:11:19 -04009891 */
9892 ret = -1;
Chris Mason4313b392008-01-03 09:08:48 -05009893
liubocdcb7252011-08-03 10:15:25 +00009894 /*
9895 * index:
9896 * 0: raid10
9897 * 1: raid1
9898 * 2: dup
9899 * 3: raid0
9900 * 4: single
9901 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009902 target = get_restripe_target(fs_info, block_group->flags);
Ilya Dryomov4a5e98f2012-03-27 17:09:17 +03009903 if (target) {
Qu Wenruo3e72ee82018-01-30 18:20:45 +08009904 index = btrfs_bg_flags_to_raid_index(extended_to_chunk(target));
Ilya Dryomov4a5e98f2012-03-27 17:09:17 +03009905 } else {
9906 /*
9907 * this is just a balance, so if we were marked as full
9908 * we know there is no space for a new chunk
9909 */
Qu Wenruo0305bc22016-03-23 11:38:17 +08009910 if (full) {
9911 if (debug)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009912 btrfs_warn(fs_info,
9913 "no space to alloc new chunk for block group %llu",
9914 block_group->key.objectid);
Ilya Dryomov4a5e98f2012-03-27 17:09:17 +03009915 goto out;
Qu Wenruo0305bc22016-03-23 11:38:17 +08009916 }
Ilya Dryomov4a5e98f2012-03-27 17:09:17 +03009917
Qu Wenruo3e72ee82018-01-30 18:20:45 +08009918 index = btrfs_bg_flags_to_raid_index(block_group->flags);
Ilya Dryomov4a5e98f2012-03-27 17:09:17 +03009919 }
9920
Miao Xiee6ec7162013-01-17 05:38:51 +00009921 if (index == BTRFS_RAID_RAID10) {
liubocdcb7252011-08-03 10:15:25 +00009922 dev_min = 4;
Josef Bacik6719db62011-08-20 08:29:51 -04009923 /* Divide by 2 */
9924 min_free >>= 1;
Miao Xiee6ec7162013-01-17 05:38:51 +00009925 } else if (index == BTRFS_RAID_RAID1) {
liubocdcb7252011-08-03 10:15:25 +00009926 dev_min = 2;
Miao Xiee6ec7162013-01-17 05:38:51 +00009927 } else if (index == BTRFS_RAID_DUP) {
Josef Bacik6719db62011-08-20 08:29:51 -04009928 /* Multiply by 2 */
9929 min_free <<= 1;
Miao Xiee6ec7162013-01-17 05:38:51 +00009930 } else if (index == BTRFS_RAID_RAID0) {
liubocdcb7252011-08-03 10:15:25 +00009931 dev_min = fs_devices->rw_devices;
David Sterba47c57132015-02-20 18:43:47 +01009932 min_free = div64_u64(min_free, dev_min);
liubocdcb7252011-08-03 10:15:25 +00009933 }
9934
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009935 mutex_lock(&fs_info->chunk_mutex);
Josef Bacikba1bf482009-09-11 16:11:19 -04009936 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
Miao Xie7bfc8372011-01-05 10:07:26 +00009937 u64 dev_offset;
Chris Masonea8c2812008-08-04 23:17:27 -04009938
Josef Bacikba1bf482009-09-11 16:11:19 -04009939 /*
9940 * check to make sure we can actually find a chunk with enough
9941 * space to fit our block group in.
9942 */
Stefan Behrens63a212a2012-11-05 18:29:28 +01009943 if (device->total_bytes > device->bytes_used + min_free &&
Anand Jain401e29c2017-12-04 12:54:55 +08009944 !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
Nikolay Borisov60dfdf22019-03-27 14:24:14 +02009945 ret = find_free_dev_extent(device, min_free,
Miao Xie7bfc8372011-01-05 10:07:26 +00009946 &dev_offset, NULL);
Josef Bacikba1bf482009-09-11 16:11:19 -04009947 if (!ret)
liubocdcb7252011-08-03 10:15:25 +00009948 dev_nr++;
9949
9950 if (dev_nr >= dev_min)
Yan73e48b22008-01-03 14:14:39 -05009951 break;
liubocdcb7252011-08-03 10:15:25 +00009952
Josef Bacikba1bf482009-09-11 16:11:19 -04009953 ret = -1;
Yan73e48b22008-01-03 14:14:39 -05009954 }
Chris Masonedbd8d42007-12-21 16:27:24 -05009955 }
Qu Wenruo0305bc22016-03-23 11:38:17 +08009956 if (debug && ret == -1)
Jeff Mahoney0b246af2016-06-22 18:54:23 -04009957 btrfs_warn(fs_info,
9958 "no space to allocate a new chunk for block group %llu",
9959 block_group->key.objectid);
9960 mutex_unlock(&fs_info->chunk_mutex);
Chris Masonedbd8d42007-12-21 16:27:24 -05009961out:
Josef Bacikba1bf482009-09-11 16:11:19 -04009962 btrfs_put_block_group(block_group);
Chris Masonedbd8d42007-12-21 16:27:24 -05009963 return ret;
9964}
9965
Jeff Mahoney6bccf3a2016-06-21 21:16:51 -04009966static int find_first_block_group(struct btrfs_fs_info *fs_info,
9967 struct btrfs_path *path,
9968 struct btrfs_key *key)
Chris Mason0b86a832008-03-24 15:01:56 -04009969{
Jeff Mahoney6bccf3a2016-06-21 21:16:51 -04009970 struct btrfs_root *root = fs_info->extent_root;
Chris Mason925baed2008-06-25 16:01:30 -04009971 int ret = 0;
Chris Mason0b86a832008-03-24 15:01:56 -04009972 struct btrfs_key found_key;
9973 struct extent_buffer *leaf;
Qu Wenruo514c7dc2018-08-01 10:37:16 +08009974 struct btrfs_block_group_item bg;
9975 u64 flags;
Chris Mason0b86a832008-03-24 15:01:56 -04009976 int slot;
9977
9978 ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
9979 if (ret < 0)
Chris Mason925baed2008-06-25 16:01:30 -04009980 goto out;
9981
Chris Masond3977122009-01-05 21:25:51 -05009982 while (1) {
Chris Mason0b86a832008-03-24 15:01:56 -04009983 slot = path->slots[0];
9984 leaf = path->nodes[0];
9985 if (slot >= btrfs_header_nritems(leaf)) {
9986 ret = btrfs_next_leaf(root, path);
9987 if (ret == 0)
9988 continue;
9989 if (ret < 0)
Chris Mason925baed2008-06-25 16:01:30 -04009990 goto out;
Chris Mason0b86a832008-03-24 15:01:56 -04009991 break;
9992 }
9993 btrfs_item_key_to_cpu(leaf, &found_key, slot);
9994
9995 if (found_key.objectid >= key->objectid &&
Chris Mason925baed2008-06-25 16:01:30 -04009996 found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
Liu Bo6fb37b72016-06-22 18:31:27 -07009997 struct extent_map_tree *em_tree;
9998 struct extent_map *em;
9999
10000 em_tree = &root->fs_info->mapping_tree.map_tree;
10001 read_lock(&em_tree->lock);
10002 em = lookup_extent_mapping(em_tree, found_key.objectid,
10003 found_key.offset);
10004 read_unlock(&em_tree->lock);
10005 if (!em) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010006 btrfs_err(fs_info,
Liu Bo6fb37b72016-06-22 18:31:27 -070010007 "logical %llu len %llu found bg but no related chunk",
10008 found_key.objectid, found_key.offset);
10009 ret = -ENOENT;
Qu Wenruo514c7dc2018-08-01 10:37:16 +080010010 } else if (em->start != found_key.objectid ||
10011 em->len != found_key.offset) {
10012 btrfs_err(fs_info,
10013 "block group %llu len %llu mismatch with chunk %llu len %llu",
10014 found_key.objectid, found_key.offset,
10015 em->start, em->len);
10016 ret = -EUCLEAN;
Liu Bo6fb37b72016-06-22 18:31:27 -070010017 } else {
Qu Wenruo514c7dc2018-08-01 10:37:16 +080010018 read_extent_buffer(leaf, &bg,
10019 btrfs_item_ptr_offset(leaf, slot),
10020 sizeof(bg));
10021 flags = btrfs_block_group_flags(&bg) &
10022 BTRFS_BLOCK_GROUP_TYPE_MASK;
10023
10024 if (flags != (em->map_lookup->type &
10025 BTRFS_BLOCK_GROUP_TYPE_MASK)) {
10026 btrfs_err(fs_info,
10027"block group %llu len %llu type flags 0x%llx mismatch with chunk type flags 0x%llx",
10028 found_key.objectid,
10029 found_key.offset, flags,
10030 (BTRFS_BLOCK_GROUP_TYPE_MASK &
10031 em->map_lookup->type));
10032 ret = -EUCLEAN;
10033 } else {
10034 ret = 0;
10035 }
Liu Bo6fb37b72016-06-22 18:31:27 -070010036 }
Josef Bacik187ee582016-08-18 15:30:06 -040010037 free_extent_map(em);
Chris Mason925baed2008-06-25 16:01:30 -040010038 goto out;
10039 }
Chris Mason0b86a832008-03-24 15:01:56 -040010040 path->slots[0]++;
10041 }
Chris Mason925baed2008-06-25 16:01:30 -040010042out:
Chris Mason0b86a832008-03-24 15:01:56 -040010043 return ret;
10044}
10045
Josef Bacik0af3d002010-06-21 14:48:16 -040010046void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
10047{
10048 struct btrfs_block_group_cache *block_group;
10049 u64 last = 0;
10050
10051 while (1) {
10052 struct inode *inode;
10053
10054 block_group = btrfs_lookup_first_block_group(info, last);
10055 while (block_group) {
Josef Bacik3aa7c7a2018-09-12 10:45:45 -040010056 wait_block_group_cache_done(block_group);
Josef Bacik0af3d002010-06-21 14:48:16 -040010057 spin_lock(&block_group->lock);
10058 if (block_group->iref)
10059 break;
10060 spin_unlock(&block_group->lock);
David Sterbaf87b7eb2019-03-20 12:01:07 +010010061 block_group = next_block_group(block_group);
Josef Bacik0af3d002010-06-21 14:48:16 -040010062 }
10063 if (!block_group) {
10064 if (last == 0)
10065 break;
10066 last = 0;
10067 continue;
10068 }
10069
10070 inode = block_group->inode;
10071 block_group->iref = 0;
10072 block_group->inode = NULL;
10073 spin_unlock(&block_group->lock);
Liu Bof3bca802016-07-20 17:33:44 -070010074 ASSERT(block_group->io_ctl.inode == NULL);
Josef Bacik0af3d002010-06-21 14:48:16 -040010075 iput(inode);
10076 last = block_group->key.objectid + block_group->key.offset;
10077 btrfs_put_block_group(block_group);
10078 }
10079}
10080
Filipe Manana5cdd7db2017-02-01 22:39:50 +000010081/*
10082 * Must be called only after stopping all workers, since we could have block
10083 * group caching kthreads running, and therefore they could race with us if we
10084 * freed the block groups before stopping them.
10085 */
Zheng Yan1a40e232008-09-26 10:09:34 -040010086int btrfs_free_block_groups(struct btrfs_fs_info *info)
10087{
10088 struct btrfs_block_group_cache *block_group;
Chris Mason4184ea72009-03-10 12:39:20 -040010089 struct btrfs_space_info *space_info;
Yan Zheng11833d62009-09-11 16:11:19 -040010090 struct btrfs_caching_control *caching_ctl;
Zheng Yan1a40e232008-09-26 10:09:34 -040010091 struct rb_node *n;
10092
Josef Bacik9e351cc2014-03-13 15:42:13 -040010093 down_write(&info->commit_root_sem);
Yan Zheng11833d62009-09-11 16:11:19 -040010094 while (!list_empty(&info->caching_block_groups)) {
10095 caching_ctl = list_entry(info->caching_block_groups.next,
10096 struct btrfs_caching_control, list);
10097 list_del(&caching_ctl->list);
10098 put_caching_control(caching_ctl);
10099 }
Josef Bacik9e351cc2014-03-13 15:42:13 -040010100 up_write(&info->commit_root_sem);
Yan Zheng11833d62009-09-11 16:11:19 -040010101
Josef Bacik47ab2a62014-09-18 11:20:02 -040010102 spin_lock(&info->unused_bgs_lock);
10103 while (!list_empty(&info->unused_bgs)) {
10104 block_group = list_first_entry(&info->unused_bgs,
10105 struct btrfs_block_group_cache,
10106 bg_list);
10107 list_del_init(&block_group->bg_list);
10108 btrfs_put_block_group(block_group);
10109 }
10110 spin_unlock(&info->unused_bgs_lock);
10111
Zheng Yan1a40e232008-09-26 10:09:34 -040010112 spin_lock(&info->block_group_cache_lock);
10113 while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
10114 block_group = rb_entry(n, struct btrfs_block_group_cache,
10115 cache_node);
Zheng Yan1a40e232008-09-26 10:09:34 -040010116 rb_erase(&block_group->cache_node,
10117 &info->block_group_cache_tree);
Filipe Manana01eacb22014-12-04 18:38:30 +000010118 RB_CLEAR_NODE(&block_group->cache_node);
Yan Zhengd899e052008-10-30 14:25:28 -040010119 spin_unlock(&info->block_group_cache_lock);
10120
Josef Bacik80eb2342008-10-29 14:49:05 -040010121 down_write(&block_group->space_info->groups_sem);
Zheng Yan1a40e232008-09-26 10:09:34 -040010122 list_del(&block_group->list);
Josef Bacik80eb2342008-10-29 14:49:05 -040010123 up_write(&block_group->space_info->groups_sem);
Yan Zhengd2fb3432008-12-11 16:30:39 -050010124
Josef Bacik3c148742011-02-02 15:53:47 +000010125 /*
10126 * We haven't cached this block group, which means we could
10127 * possibly have excluded extents on this block group.
10128 */
Josef Bacik36cce922013-08-05 11:15:21 -040010129 if (block_group->cached == BTRFS_CACHE_NO ||
10130 block_group->cached == BTRFS_CACHE_ERROR)
Nikolay Borisov9e715da2018-06-20 15:49:08 +030010131 free_excluded_extents(block_group);
Josef Bacik3c148742011-02-02 15:53:47 +000010132
Josef Bacik817d52f2009-07-13 21:29:25 -040010133 btrfs_remove_free_space_cache(block_group);
Filipe Manana5cdd7db2017-02-01 22:39:50 +000010134 ASSERT(block_group->cached != BTRFS_CACHE_STARTED);
Liu Bof3bca802016-07-20 17:33:44 -070010135 ASSERT(list_empty(&block_group->dirty_list));
10136 ASSERT(list_empty(&block_group->io_list));
10137 ASSERT(list_empty(&block_group->bg_list));
10138 ASSERT(atomic_read(&block_group->count) == 1);
Josef Bacik11dfe352009-11-13 20:12:59 +000010139 btrfs_put_block_group(block_group);
Yan Zhengd899e052008-10-30 14:25:28 -040010140
10141 spin_lock(&info->block_group_cache_lock);
Zheng Yan1a40e232008-09-26 10:09:34 -040010142 }
10143 spin_unlock(&info->block_group_cache_lock);
Chris Mason4184ea72009-03-10 12:39:20 -040010144
10145 /* now that all the block groups are freed, go through and
10146 * free all the space_info structs. This is only called during
10147 * the final stages of unmount, and so we know nobody is
10148 * using them. We call synchronize_rcu() once before we start,
10149 * just to be on the safe side.
10150 */
10151 synchronize_rcu();
10152
Yan, Zheng8929ecfa2010-05-16 10:49:58 -040010153 release_global_block_rsv(info);
10154
Dulshani Gunawardhana67871252013-10-31 10:33:04 +053010155 while (!list_empty(&info->space_info)) {
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040010156 int i;
10157
Chris Mason4184ea72009-03-10 12:39:20 -040010158 space_info = list_entry(info->space_info.next,
10159 struct btrfs_space_info,
10160 list);
Josef Bacikd555b6c2016-03-25 13:25:51 -040010161
10162 /*
10163 * Do not hide this behind enospc_debug, this is actually
10164 * important and indicates a real bug if this happens.
10165 */
10166 if (WARN_ON(space_info->bytes_pinned > 0 ||
David Sterbab069e0c2013-02-08 21:28:17 +000010167 space_info->bytes_reserved > 0 ||
Josef Bacikd555b6c2016-03-25 13:25:51 -040010168 space_info->bytes_may_use > 0))
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -040010169 dump_space_info(info, space_info, 0, 0);
Chris Mason4184ea72009-03-10 12:39:20 -040010170 list_del(&space_info->list);
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040010171 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
10172 struct kobject *kobj;
Jeff Mahoneyc1895442014-05-27 12:59:57 -040010173 kobj = space_info->block_group_kobjs[i];
10174 space_info->block_group_kobjs[i] = NULL;
10175 if (kobj) {
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040010176 kobject_del(kobj);
10177 kobject_put(kobj);
10178 }
10179 }
10180 kobject_del(&space_info->kobj);
10181 kobject_put(&space_info->kobj);
Chris Mason4184ea72009-03-10 12:39:20 -040010182 }
Zheng Yan1a40e232008-09-26 10:09:34 -040010183 return 0;
10184}
10185
Jeff Mahoney75cb3792018-03-20 15:25:26 -040010186/* link_block_group will queue up kobjects to add when we're reclaim-safe */
10187void btrfs_add_raid_kobjects(struct btrfs_fs_info *fs_info)
10188{
10189 struct btrfs_space_info *space_info;
10190 struct raid_kobject *rkobj;
10191 LIST_HEAD(list);
10192 int index;
10193 int ret = 0;
10194
10195 spin_lock(&fs_info->pending_raid_kobjs_lock);
10196 list_splice_init(&fs_info->pending_raid_kobjs, &list);
10197 spin_unlock(&fs_info->pending_raid_kobjs_lock);
10198
10199 list_for_each_entry(rkobj, &list, list) {
10200 space_info = __find_space_info(fs_info, rkobj->flags);
10201 index = btrfs_bg_flags_to_raid_index(rkobj->flags);
10202
10203 ret = kobject_add(&rkobj->kobj, &space_info->kobj,
10204 "%s", get_raid_name(index));
10205 if (ret) {
10206 kobject_put(&rkobj->kobj);
10207 break;
10208 }
10209 }
10210 if (ret)
10211 btrfs_warn(fs_info,
10212 "failed to add kobject for block cache, ignoring");
10213}
10214
Nikolay Borisovc434d212017-08-21 12:43:50 +030010215static void link_block_group(struct btrfs_block_group_cache *cache)
Yan, Zhengb742bb822010-05-16 10:46:24 -040010216{
Nikolay Borisovc434d212017-08-21 12:43:50 +030010217 struct btrfs_space_info *space_info = cache->space_info;
Jeff Mahoney75cb3792018-03-20 15:25:26 -040010218 struct btrfs_fs_info *fs_info = cache->fs_info;
Qu Wenruo3e72ee82018-01-30 18:20:45 +080010219 int index = btrfs_bg_flags_to_raid_index(cache->flags);
Jeff Mahoneyed55b6a2014-03-26 14:11:26 -040010220 bool first = false;
Yan, Zhengb742bb822010-05-16 10:46:24 -040010221
10222 down_write(&space_info->groups_sem);
Jeff Mahoneyed55b6a2014-03-26 14:11:26 -040010223 if (list_empty(&space_info->block_groups[index]))
10224 first = true;
10225 list_add_tail(&cache->list, &space_info->block_groups[index]);
10226 up_write(&space_info->groups_sem);
10227
10228 if (first) {
Jeff Mahoney75cb3792018-03-20 15:25:26 -040010229 struct raid_kobject *rkobj = kzalloc(sizeof(*rkobj), GFP_NOFS);
10230 if (!rkobj) {
10231 btrfs_warn(cache->fs_info,
10232 "couldn't alloc memory for raid level kobject");
10233 return;
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040010234 }
Jeff Mahoney75cb3792018-03-20 15:25:26 -040010235 rkobj->flags = cache->flags;
10236 kobject_init(&rkobj->kobj, &btrfs_raid_ktype);
10237
10238 spin_lock(&fs_info->pending_raid_kobjs_lock);
10239 list_add_tail(&rkobj->list, &fs_info->pending_raid_kobjs);
10240 spin_unlock(&fs_info->pending_raid_kobjs_lock);
Jeff Mahoneyc1895442014-05-27 12:59:57 -040010241 space_info->block_group_kobjs[index] = &rkobj->kobj;
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040010242 }
Yan, Zhengb742bb822010-05-16 10:46:24 -040010243}
10244
Miao Xie920e4a52014-01-15 20:00:55 +080010245static struct btrfs_block_group_cache *
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040010246btrfs_create_block_group_cache(struct btrfs_fs_info *fs_info,
10247 u64 start, u64 size)
Miao Xie920e4a52014-01-15 20:00:55 +080010248{
10249 struct btrfs_block_group_cache *cache;
10250
10251 cache = kzalloc(sizeof(*cache), GFP_NOFS);
10252 if (!cache)
10253 return NULL;
10254
10255 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
10256 GFP_NOFS);
10257 if (!cache->free_space_ctl) {
10258 kfree(cache);
10259 return NULL;
10260 }
10261
10262 cache->key.objectid = start;
10263 cache->key.offset = size;
10264 cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
10265
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010266 cache->fs_info = fs_info;
Nikolay Borisove4ff5fb2017-07-19 10:48:42 +030010267 cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
Omar Sandoval1e144fb2015-09-29 20:50:37 -070010268 set_free_space_tree_thresholds(cache);
10269
Miao Xie920e4a52014-01-15 20:00:55 +080010270 atomic_set(&cache->count, 1);
10271 spin_lock_init(&cache->lock);
Miao Xiee570fd22014-06-19 10:42:50 +080010272 init_rwsem(&cache->data_rwsem);
Miao Xie920e4a52014-01-15 20:00:55 +080010273 INIT_LIST_HEAD(&cache->list);
10274 INIT_LIST_HEAD(&cache->cluster_list);
Josef Bacik47ab2a62014-09-18 11:20:02 -040010275 INIT_LIST_HEAD(&cache->bg_list);
Josef Bacik633c0aa2014-10-31 09:49:34 -040010276 INIT_LIST_HEAD(&cache->ro_list);
Josef Bacikce93ec52014-11-17 15:45:48 -050010277 INIT_LIST_HEAD(&cache->dirty_list);
Chris Masonc9dc4c62015-04-04 17:14:42 -070010278 INIT_LIST_HEAD(&cache->io_list);
Miao Xie920e4a52014-01-15 20:00:55 +080010279 btrfs_init_free_space_ctl(cache);
Filipe Manana04216822014-11-27 21:14:15 +000010280 atomic_set(&cache->trimming, 0);
Omar Sandovala5ed9182015-09-29 20:50:35 -070010281 mutex_init(&cache->free_space_lock);
Qu Wenruo0966a7b2017-04-14 08:35:54 +080010282 btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root);
Miao Xie920e4a52014-01-15 20:00:55 +080010283
10284 return cache;
10285}
10286
Qu Wenruo7ef49512018-08-01 10:37:17 +080010287
10288/*
10289 * Iterate all chunks and verify that each of them has the corresponding block
10290 * group
10291 */
10292static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info)
10293{
10294 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
10295 struct extent_map *em;
10296 struct btrfs_block_group_cache *bg;
10297 u64 start = 0;
10298 int ret = 0;
10299
10300 while (1) {
10301 read_lock(&map_tree->map_tree.lock);
10302 /*
10303 * lookup_extent_mapping will return the first extent map
10304 * intersecting the range, so setting @len to 1 is enough to
10305 * get the first chunk.
10306 */
10307 em = lookup_extent_mapping(&map_tree->map_tree, start, 1);
10308 read_unlock(&map_tree->map_tree.lock);
10309 if (!em)
10310 break;
10311
10312 bg = btrfs_lookup_block_group(fs_info, em->start);
10313 if (!bg) {
10314 btrfs_err(fs_info,
10315 "chunk start=%llu len=%llu doesn't have corresponding block group",
10316 em->start, em->len);
10317 ret = -EUCLEAN;
10318 free_extent_map(em);
10319 break;
10320 }
10321 if (bg->key.objectid != em->start ||
10322 bg->key.offset != em->len ||
10323 (bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) !=
10324 (em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
10325 btrfs_err(fs_info,
10326"chunk start=%llu len=%llu flags=0x%llx doesn't match block group start=%llu len=%llu flags=0x%llx",
10327 em->start, em->len,
10328 em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK,
10329 bg->key.objectid, bg->key.offset,
10330 bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK);
10331 ret = -EUCLEAN;
10332 free_extent_map(em);
10333 btrfs_put_block_group(bg);
10334 break;
10335 }
10336 start = em->start + em->len;
10337 free_extent_map(em);
10338 btrfs_put_block_group(bg);
10339 }
10340 return ret;
10341}
10342
Jeff Mahoney5b4aace2016-06-21 10:40:19 -040010343int btrfs_read_block_groups(struct btrfs_fs_info *info)
Chris Mason9078a3e2007-04-26 16:46:15 -040010344{
10345 struct btrfs_path *path;
10346 int ret;
Chris Mason9078a3e2007-04-26 16:46:15 -040010347 struct btrfs_block_group_cache *cache;
Chris Mason6324fbf2008-03-24 15:01:59 -040010348 struct btrfs_space_info *space_info;
Chris Mason9078a3e2007-04-26 16:46:15 -040010349 struct btrfs_key key;
10350 struct btrfs_key found_key;
Chris Mason5f39d392007-10-15 16:14:19 -040010351 struct extent_buffer *leaf;
Josef Bacik0af3d002010-06-21 14:48:16 -040010352 int need_clear = 0;
10353 u64 cache_gen;
Liu Bo49303382016-08-25 18:08:27 -070010354 u64 feature;
10355 int mixed;
10356
10357 feature = btrfs_super_incompat_flags(info->super_copy);
10358 mixed = !!(feature & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS);
Chris Mason96b51792007-10-15 16:15:19 -040010359
Chris Mason9078a3e2007-04-26 16:46:15 -040010360 key.objectid = 0;
Chris Mason0b86a832008-03-24 15:01:56 -040010361 key.offset = 0;
David Sterba962a2982014-06-04 18:41:45 +020010362 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
Chris Mason9078a3e2007-04-26 16:46:15 -040010363 path = btrfs_alloc_path();
10364 if (!path)
10365 return -ENOMEM;
David Sterbae4058b52015-11-27 16:31:35 +010010366 path->reada = READA_FORWARD;
Chris Mason9078a3e2007-04-26 16:46:15 -040010367
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010368 cache_gen = btrfs_super_cache_generation(info->super_copy);
10369 if (btrfs_test_opt(info, SPACE_CACHE) &&
10370 btrfs_super_generation(info->super_copy) != cache_gen)
Josef Bacik0af3d002010-06-21 14:48:16 -040010371 need_clear = 1;
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010372 if (btrfs_test_opt(info, CLEAR_CACHE))
Josef Bacik88c2ba32010-09-21 14:21:34 -040010373 need_clear = 1;
Josef Bacik0af3d002010-06-21 14:48:16 -040010374
Chris Masond3977122009-01-05 21:25:51 -050010375 while (1) {
Jeff Mahoney6bccf3a2016-06-21 21:16:51 -040010376 ret = find_first_block_group(info, path, &key);
Yan, Zhengb742bb822010-05-16 10:46:24 -040010377 if (ret > 0)
10378 break;
Chris Mason0b86a832008-03-24 15:01:56 -040010379 if (ret != 0)
10380 goto error;
Miao Xie920e4a52014-01-15 20:00:55 +080010381
Chris Mason5f39d392007-10-15 16:14:19 -040010382 leaf = path->nodes[0];
10383 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
Miao Xie920e4a52014-01-15 20:00:55 +080010384
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040010385 cache = btrfs_create_block_group_cache(info, found_key.objectid,
Miao Xie920e4a52014-01-15 20:00:55 +080010386 found_key.offset);
Chris Mason9078a3e2007-04-26 16:46:15 -040010387 if (!cache) {
Chris Mason0b86a832008-03-24 15:01:56 -040010388 ret = -ENOMEM;
Yan, Zhengf0486c62010-05-16 10:46:25 -040010389 goto error;
Chris Mason9078a3e2007-04-26 16:46:15 -040010390 }
Josef Bacik96303082009-07-13 21:29:25 -040010391
Liu Bocf7c1ef2012-07-06 03:31:34 -060010392 if (need_clear) {
10393 /*
10394 * When we mount with old space cache, we need to
10395 * set BTRFS_DC_CLEAR and set dirty flag.
10396 *
10397 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
10398 * truncate the old free space cache inode and
10399 * setup a new one.
10400 * b) Setting 'dirty flag' makes sure that we flush
10401 * the new space cache info onto disk.
10402 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010403 if (btrfs_test_opt(info, SPACE_CACHE))
Josef Bacikce93ec52014-11-17 15:45:48 -050010404 cache->disk_cache_state = BTRFS_DC_CLEAR;
Liu Bocf7c1ef2012-07-06 03:31:34 -060010405 }
Josef Bacik0af3d002010-06-21 14:48:16 -040010406
Chris Mason5f39d392007-10-15 16:14:19 -040010407 read_extent_buffer(leaf, &cache->item,
10408 btrfs_item_ptr_offset(leaf, path->slots[0]),
10409 sizeof(cache->item));
Miao Xie920e4a52014-01-15 20:00:55 +080010410 cache->flags = btrfs_block_group_flags(&cache->item);
Liu Bo49303382016-08-25 18:08:27 -070010411 if (!mixed &&
10412 ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) &&
10413 (cache->flags & BTRFS_BLOCK_GROUP_DATA))) {
10414 btrfs_err(info,
10415"bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups",
10416 cache->key.objectid);
10417 ret = -EINVAL;
10418 goto error;
10419 }
Chris Mason0b86a832008-03-24 15:01:56 -040010420
Chris Mason9078a3e2007-04-26 16:46:15 -040010421 key.objectid = found_key.objectid + found_key.offset;
David Sterbab3b4aa72011-04-21 01:20:15 +020010422 btrfs_release_path(path);
Li Zefan34d52cb2011-03-29 13:46:06 +080010423
Josef Bacik817d52f2009-07-13 21:29:25 -040010424 /*
Josef Bacik3c148742011-02-02 15:53:47 +000010425 * We need to exclude the super stripes now so that the space
10426 * info has super bytes accounted for, otherwise we'll think
10427 * we have more space than we actually do.
10428 */
Nikolay Borisov3c4da652018-06-20 15:49:09 +030010429 ret = exclude_super_stripes(cache);
Josef Bacik835d9742013-03-19 12:13:25 -040010430 if (ret) {
10431 /*
10432 * We may have excluded something, so call this just in
10433 * case.
10434 */
Nikolay Borisov9e715da2018-06-20 15:49:08 +030010435 free_excluded_extents(cache);
Miao Xie920e4a52014-01-15 20:00:55 +080010436 btrfs_put_block_group(cache);
Josef Bacik835d9742013-03-19 12:13:25 -040010437 goto error;
10438 }
Josef Bacik3c148742011-02-02 15:53:47 +000010439
10440 /*
Josef Bacik817d52f2009-07-13 21:29:25 -040010441 * check for two cases, either we are full, and therefore
10442 * don't need to bother with the caching work since we won't
10443 * find any space, or we are empty, and we can just add all
Andrea Gelmini52042d82018-11-28 12:05:13 +010010444 * the space in and be done with it. This saves us _a_lot_ of
Josef Bacik817d52f2009-07-13 21:29:25 -040010445 * time, particularly in the full case.
10446 */
10447 if (found_key.offset == btrfs_block_group_used(&cache->item)) {
Yan Zheng11833d62009-09-11 16:11:19 -040010448 cache->last_byte_to_unpin = (u64)-1;
Josef Bacik817d52f2009-07-13 21:29:25 -040010449 cache->cached = BTRFS_CACHE_FINISHED;
Nikolay Borisov9e715da2018-06-20 15:49:08 +030010450 free_excluded_extents(cache);
Josef Bacik817d52f2009-07-13 21:29:25 -040010451 } else if (btrfs_block_group_used(&cache->item) == 0) {
Yan Zheng11833d62009-09-11 16:11:19 -040010452 cache->last_byte_to_unpin = (u64)-1;
Josef Bacik817d52f2009-07-13 21:29:25 -040010453 cache->cached = BTRFS_CACHE_FINISHED;
Nikolay Borisov4457c1c2018-05-10 15:44:45 +030010454 add_new_free_space(cache, found_key.objectid,
Josef Bacik817d52f2009-07-13 21:29:25 -040010455 found_key.objectid +
10456 found_key.offset);
Nikolay Borisov9e715da2018-06-20 15:49:08 +030010457 free_excluded_extents(cache);
Josef Bacik817d52f2009-07-13 21:29:25 -040010458 }
Chris Mason96b51792007-10-15 16:15:19 -040010459
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010460 ret = btrfs_add_block_group_cache(info, cache);
Josef Bacik8c579fe2013-04-02 12:40:42 -040010461 if (ret) {
10462 btrfs_remove_free_space_cache(cache);
10463 btrfs_put_block_group(cache);
10464 goto error;
10465 }
10466
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010467 trace_btrfs_add_block_group(info, cache, 0);
Nikolay Borisovd2006e62017-05-22 09:35:50 +030010468 update_space_info(info, cache->flags, found_key.offset,
10469 btrfs_block_group_used(&cache->item),
10470 cache->bytes_super, &space_info);
Josef Bacik8c579fe2013-04-02 12:40:42 -040010471
Chris Mason6324fbf2008-03-24 15:01:59 -040010472 cache->space_info = space_info;
Josef Bacik1b2da372009-09-11 16:11:20 -040010473
Nikolay Borisovc434d212017-08-21 12:43:50 +030010474 link_block_group(cache);
Chris Mason6324fbf2008-03-24 15:01:59 -040010475
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010476 set_avail_alloc_bits(info, cache->flags);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040010477 if (btrfs_chunk_readonly(info, cache->key.objectid)) {
Zhaolei868f4012015-08-05 16:43:27 +080010478 inc_block_group_ro(cache, 1);
Josef Bacik47ab2a62014-09-18 11:20:02 -040010479 } else if (btrfs_block_group_used(&cache->item) == 0) {
Qu Wenruo031f24d2018-05-22 16:43:47 +080010480 ASSERT(list_empty(&cache->bg_list));
10481 btrfs_mark_bg_unused(cache);
Josef Bacik47ab2a62014-09-18 11:20:02 -040010482 }
Chris Mason9078a3e2007-04-26 16:46:15 -040010483 }
Yan, Zhengb742bb822010-05-16 10:46:24 -040010484
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010485 list_for_each_entry_rcu(space_info, &info->space_info, list) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040010486 if (!(get_alloc_profile(info, space_info->flags) &
Yan, Zhengb742bb822010-05-16 10:46:24 -040010487 (BTRFS_BLOCK_GROUP_RAID10 |
10488 BTRFS_BLOCK_GROUP_RAID1 |
David Woodhouse53b381b2013-01-29 18:40:14 -050010489 BTRFS_BLOCK_GROUP_RAID5 |
10490 BTRFS_BLOCK_GROUP_RAID6 |
Yan, Zhengb742bb822010-05-16 10:46:24 -040010491 BTRFS_BLOCK_GROUP_DUP)))
10492 continue;
10493 /*
10494 * avoid allocating from un-mirrored block group if there are
10495 * mirrored block groups.
10496 */
chandan1095cc02013-07-16 12:28:56 +053010497 list_for_each_entry(cache,
10498 &space_info->block_groups[BTRFS_RAID_RAID0],
10499 list)
Zhaolei868f4012015-08-05 16:43:27 +080010500 inc_block_group_ro(cache, 1);
chandan1095cc02013-07-16 12:28:56 +053010501 list_for_each_entry(cache,
10502 &space_info->block_groups[BTRFS_RAID_SINGLE],
10503 list)
Zhaolei868f4012015-08-05 16:43:27 +080010504 inc_block_group_ro(cache, 1);
Yan, Zhengb742bb822010-05-16 10:46:24 -040010505 }
Yan, Zhengf0486c62010-05-16 10:46:25 -040010506
Jeff Mahoney75cb3792018-03-20 15:25:26 -040010507 btrfs_add_raid_kobjects(info);
Yan, Zhengf0486c62010-05-16 10:46:25 -040010508 init_global_block_rsv(info);
Qu Wenruo7ef49512018-08-01 10:37:17 +080010509 ret = check_chunk_block_group_mappings(info);
Chris Mason0b86a832008-03-24 15:01:56 -040010510error:
Chris Mason9078a3e2007-04-26 16:46:15 -040010511 btrfs_free_path(path);
Chris Mason0b86a832008-03-24 15:01:56 -040010512 return ret;
Chris Mason9078a3e2007-04-26 16:46:15 -040010513}
Chris Mason6324fbf2008-03-24 15:01:59 -040010514
Nikolay Borisov6c686b32018-02-07 17:55:40 +020010515void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans)
Josef Bacikea658ba2012-09-11 16:57:25 -040010516{
Nikolay Borisov6c686b32018-02-07 17:55:40 +020010517 struct btrfs_fs_info *fs_info = trans->fs_info;
Josef Bacik545e3362018-09-28 07:18:02 -040010518 struct btrfs_block_group_cache *block_group;
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010519 struct btrfs_root *extent_root = fs_info->extent_root;
Josef Bacikea658ba2012-09-11 16:57:25 -040010520 struct btrfs_block_group_item item;
10521 struct btrfs_key key;
10522 int ret = 0;
10523
Filipe Manana5ce55552018-10-12 10:03:55 +010010524 if (!trans->can_flush_pending_bgs)
10525 return;
10526
Josef Bacik545e3362018-09-28 07:18:02 -040010527 while (!list_empty(&trans->new_bgs)) {
10528 block_group = list_first_entry(&trans->new_bgs,
10529 struct btrfs_block_group_cache,
10530 bg_list);
Josef Bacikea658ba2012-09-11 16:57:25 -040010531 if (ret)
Filipe Mananac92f6be2014-11-26 15:28:55 +000010532 goto next;
Josef Bacikea658ba2012-09-11 16:57:25 -040010533
10534 spin_lock(&block_group->lock);
10535 memcpy(&item, &block_group->item, sizeof(item));
10536 memcpy(&key, &block_group->key, sizeof(key));
10537 spin_unlock(&block_group->lock);
10538
10539 ret = btrfs_insert_item(trans, extent_root, &key, &item,
10540 sizeof(item));
10541 if (ret)
Jeff Mahoney66642832016-06-10 18:19:25 -040010542 btrfs_abort_transaction(trans, ret);
Nikolay Borisov97aff912018-07-20 19:37:53 +030010543 ret = btrfs_finish_chunk_alloc(trans, key.objectid, key.offset);
Josef Bacik6df9a952013-06-27 13:22:46 -040010544 if (ret)
Jeff Mahoney66642832016-06-10 18:19:25 -040010545 btrfs_abort_transaction(trans, ret);
Nikolay Borisove4e07112018-05-10 15:44:41 +030010546 add_block_group_free_space(trans, block_group);
Omar Sandoval1e144fb2015-09-29 20:50:37 -070010547 /* already aborted the transaction if it failed. */
Filipe Mananac92f6be2014-11-26 15:28:55 +000010548next:
Josef Bacikba2c4d42018-12-03 10:20:33 -050010549 btrfs_delayed_refs_rsv_release(fs_info, 1);
Filipe Mananac92f6be2014-11-26 15:28:55 +000010550 list_del_init(&block_group->bg_list);
Josef Bacikea658ba2012-09-11 16:57:25 -040010551 }
Filipe Manana5ce55552018-10-12 10:03:55 +010010552 btrfs_trans_release_chunk_metadata(trans);
Josef Bacikea658ba2012-09-11 16:57:25 -040010553}
10554
Nikolay Borisove7e02092018-06-20 15:48:55 +030010555int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used,
Nikolay Borisov01744842017-07-27 14:22:11 +030010556 u64 type, u64 chunk_offset, u64 size)
Chris Mason6324fbf2008-03-24 15:01:59 -040010557{
Nikolay Borisove7e02092018-06-20 15:48:55 +030010558 struct btrfs_fs_info *fs_info = trans->fs_info;
Chris Mason6324fbf2008-03-24 15:01:59 -040010559 struct btrfs_block_group_cache *cache;
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010560 int ret;
Chris Mason6324fbf2008-03-24 15:01:59 -040010561
David Sterba90787762019-03-20 13:28:05 +010010562 btrfs_set_log_full_commit(trans);
Chris Masone02119d2008-09-05 16:13:11 -040010563
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040010564 cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size);
Josef Bacik0f9dd462008-09-23 13:14:11 -040010565 if (!cache)
10566 return -ENOMEM;
Li Zefan34d52cb2011-03-29 13:46:06 +080010567
Chris Mason6324fbf2008-03-24 15:01:59 -040010568 btrfs_set_block_group_used(&cache->item, bytes_used);
Nikolay Borisov01744842017-07-27 14:22:11 +030010569 btrfs_set_block_group_chunk_objectid(&cache->item,
10570 BTRFS_FIRST_CHUNK_TREE_OBJECTID);
Chris Mason6324fbf2008-03-24 15:01:59 -040010571 btrfs_set_block_group_flags(&cache->item, type);
10572
Miao Xie920e4a52014-01-15 20:00:55 +080010573 cache->flags = type;
Yan Zheng11833d62009-09-11 16:11:19 -040010574 cache->last_byte_to_unpin = (u64)-1;
Josef Bacik817d52f2009-07-13 21:29:25 -040010575 cache->cached = BTRFS_CACHE_FINISHED;
Omar Sandoval1e144fb2015-09-29 20:50:37 -070010576 cache->needs_free_space = 1;
Nikolay Borisov3c4da652018-06-20 15:49:09 +030010577 ret = exclude_super_stripes(cache);
Josef Bacik835d9742013-03-19 12:13:25 -040010578 if (ret) {
10579 /*
10580 * We may have excluded something, so call this just in
10581 * case.
10582 */
Nikolay Borisov9e715da2018-06-20 15:49:08 +030010583 free_excluded_extents(cache);
Miao Xie920e4a52014-01-15 20:00:55 +080010584 btrfs_put_block_group(cache);
Josef Bacik835d9742013-03-19 12:13:25 -040010585 return ret;
10586 }
Josef Bacik96303082009-07-13 21:29:25 -040010587
Nikolay Borisov4457c1c2018-05-10 15:44:45 +030010588 add_new_free_space(cache, chunk_offset, chunk_offset + size);
Josef Bacik817d52f2009-07-13 21:29:25 -040010589
Nikolay Borisov9e715da2018-06-20 15:49:08 +030010590 free_excluded_extents(cache);
Yan Zheng11833d62009-09-11 16:11:19 -040010591
Josef Bacikd0bd4562015-09-23 14:54:14 -040010592#ifdef CONFIG_BTRFS_DEBUG
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040010593 if (btrfs_should_fragment_free_space(cache)) {
Josef Bacikd0bd4562015-09-23 14:54:14 -040010594 u64 new_bytes_used = size - bytes_used;
10595
10596 bytes_used += new_bytes_used >> 1;
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040010597 fragment_free_space(cache);
Josef Bacikd0bd4562015-09-23 14:54:14 -040010598 }
10599#endif
Filipe Manana2e6e5182015-05-12 00:28:11 +010010600 /*
Nikolay Borisov2be12ef2017-05-22 09:35:49 +030010601 * Ensure the corresponding space_info object is created and
10602 * assigned to our block group. We want our bg to be added to the rbtree
10603 * with its ->space_info set.
Filipe Manana2e6e5182015-05-12 00:28:11 +010010604 */
Nikolay Borisov2be12ef2017-05-22 09:35:49 +030010605 cache->space_info = __find_space_info(fs_info, cache->flags);
Jeff Mahoneydc2d3002018-03-20 15:25:25 -040010606 ASSERT(cache->space_info);
Filipe Manana2e6e5182015-05-12 00:28:11 +010010607
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010608 ret = btrfs_add_block_group_cache(fs_info, cache);
Josef Bacik8c579fe2013-04-02 12:40:42 -040010609 if (ret) {
10610 btrfs_remove_free_space_cache(cache);
10611 btrfs_put_block_group(cache);
10612 return ret;
10613 }
10614
Filipe Manana2e6e5182015-05-12 00:28:11 +010010615 /*
10616 * Now that our block group has its ->space_info set and is inserted in
10617 * the rbtree, update the space info's counters.
10618 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010619 trace_btrfs_add_block_group(fs_info, cache, 1);
Nikolay Borisovd2006e62017-05-22 09:35:50 +030010620 update_space_info(fs_info, cache->flags, size, bytes_used,
Josef Bacike40edf22016-03-25 13:25:47 -040010621 cache->bytes_super, &cache->space_info);
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010622 update_global_block_rsv(fs_info);
Josef Bacik1b2da372009-09-11 16:11:20 -040010623
Nikolay Borisovc434d212017-08-21 12:43:50 +030010624 link_block_group(cache);
Chris Mason6324fbf2008-03-24 15:01:59 -040010625
Josef Bacik47ab2a62014-09-18 11:20:02 -040010626 list_add_tail(&cache->bg_list, &trans->new_bgs);
Josef Bacikba2c4d42018-12-03 10:20:33 -050010627 trans->delayed_ref_updates++;
10628 btrfs_update_delayed_refs_rsv(trans);
Chris Mason6324fbf2008-03-24 15:01:59 -040010629
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010630 set_avail_alloc_bits(fs_info, type);
Chris Mason6324fbf2008-03-24 15:01:59 -040010631 return 0;
10632}
Zheng Yan1a40e232008-09-26 10:09:34 -040010633
Ilya Dryomov10ea00f2012-01-16 22:04:47 +020010634static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
10635{
Ilya Dryomov899c81e2012-03-27 17:09:16 +030010636 u64 extra_flags = chunk_to_extended(flags) &
10637 BTRFS_EXTENDED_PROFILE_MASK;
Ilya Dryomov10ea00f2012-01-16 22:04:47 +020010638
Miao Xiede98ced2013-01-29 10:13:12 +000010639 write_seqlock(&fs_info->profiles_lock);
Ilya Dryomov10ea00f2012-01-16 22:04:47 +020010640 if (flags & BTRFS_BLOCK_GROUP_DATA)
10641 fs_info->avail_data_alloc_bits &= ~extra_flags;
10642 if (flags & BTRFS_BLOCK_GROUP_METADATA)
10643 fs_info->avail_metadata_alloc_bits &= ~extra_flags;
10644 if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
10645 fs_info->avail_system_alloc_bits &= ~extra_flags;
Miao Xiede98ced2013-01-29 10:13:12 +000010646 write_sequnlock(&fs_info->profiles_lock);
Ilya Dryomov10ea00f2012-01-16 22:04:47 +020010647}
10648
Zheng Yan1a40e232008-09-26 10:09:34 -040010649int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
Nikolay Borisov5a98ec02018-06-20 15:48:56 +030010650 u64 group_start, struct extent_map *em)
Zheng Yan1a40e232008-09-26 10:09:34 -040010651{
Nikolay Borisov5a98ec02018-06-20 15:48:56 +030010652 struct btrfs_fs_info *fs_info = trans->fs_info;
Jeff Mahoney6bccf3a2016-06-21 21:16:51 -040010653 struct btrfs_root *root = fs_info->extent_root;
Zheng Yan1a40e232008-09-26 10:09:34 -040010654 struct btrfs_path *path;
10655 struct btrfs_block_group_cache *block_group;
Chris Mason44fb5512009-06-04 15:34:51 -040010656 struct btrfs_free_cluster *cluster;
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010657 struct btrfs_root *tree_root = fs_info->tree_root;
Zheng Yan1a40e232008-09-26 10:09:34 -040010658 struct btrfs_key key;
Josef Bacik0af3d002010-06-21 14:48:16 -040010659 struct inode *inode;
Jeff Mahoneyc1895442014-05-27 12:59:57 -040010660 struct kobject *kobj = NULL;
Zheng Yan1a40e232008-09-26 10:09:34 -040010661 int ret;
Ilya Dryomov10ea00f2012-01-16 22:04:47 +020010662 int index;
Josef Bacik89a55892010-10-14 14:52:27 -040010663 int factor;
Filipe Manana4f69cb92014-11-26 15:28:51 +000010664 struct btrfs_caching_control *caching_ctl = NULL;
Filipe Manana04216822014-11-27 21:14:15 +000010665 bool remove_em;
Josef Bacikba2c4d42018-12-03 10:20:33 -050010666 bool remove_rsv = false;
Zheng Yan1a40e232008-09-26 10:09:34 -040010667
Jeff Mahoney6bccf3a2016-06-21 21:16:51 -040010668 block_group = btrfs_lookup_block_group(fs_info, group_start);
Zheng Yan1a40e232008-09-26 10:09:34 -040010669 BUG_ON(!block_group);
Yan Zhengc146afa2008-11-12 14:34:12 -050010670 BUG_ON(!block_group->ro);
Zheng Yan1a40e232008-09-26 10:09:34 -040010671
Qu Wenruo4ed0a7a2018-04-26 17:17:20 +080010672 trace_btrfs_remove_block_group(block_group);
liubo9f7c43c2011-03-07 02:13:33 +000010673 /*
10674 * Free the reserved super bytes from this block group before
10675 * remove it.
10676 */
Nikolay Borisov9e715da2018-06-20 15:49:08 +030010677 free_excluded_extents(block_group);
Josef Bacikfd708b82017-09-29 15:43:50 -040010678 btrfs_free_ref_tree_range(fs_info, block_group->key.objectid,
10679 block_group->key.offset);
liubo9f7c43c2011-03-07 02:13:33 +000010680
Zheng Yan1a40e232008-09-26 10:09:34 -040010681 memcpy(&key, &block_group->key, sizeof(key));
Qu Wenruo3e72ee82018-01-30 18:20:45 +080010682 index = btrfs_bg_flags_to_raid_index(block_group->flags);
David Sterba46df06b2018-07-13 20:46:30 +020010683 factor = btrfs_bg_type_to_factor(block_group->flags);
Zheng Yan1a40e232008-09-26 10:09:34 -040010684
Chris Mason44fb5512009-06-04 15:34:51 -040010685 /* make sure this block group isn't part of an allocation cluster */
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010686 cluster = &fs_info->data_alloc_cluster;
Chris Mason44fb5512009-06-04 15:34:51 -040010687 spin_lock(&cluster->refill_lock);
10688 btrfs_return_cluster_to_free_space(block_group, cluster);
10689 spin_unlock(&cluster->refill_lock);
10690
10691 /*
10692 * make sure this block group isn't part of a metadata
10693 * allocation cluster
10694 */
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010695 cluster = &fs_info->meta_alloc_cluster;
Chris Mason44fb5512009-06-04 15:34:51 -040010696 spin_lock(&cluster->refill_lock);
10697 btrfs_return_cluster_to_free_space(block_group, cluster);
10698 spin_unlock(&cluster->refill_lock);
10699
Zheng Yan1a40e232008-09-26 10:09:34 -040010700 path = btrfs_alloc_path();
Mark Fashehd8926bb2011-07-13 10:38:47 -070010701 if (!path) {
10702 ret = -ENOMEM;
10703 goto out;
10704 }
Zheng Yan1a40e232008-09-26 10:09:34 -040010705
Chris Mason1bbc6212015-04-06 12:46:08 -070010706 /*
10707 * get the inode first so any iput calls done for the io_list
10708 * aren't the final iput (no unlinks allowed now)
10709 */
David Sterba7949f332019-03-20 13:40:19 +010010710 inode = lookup_free_space_inode(block_group, path);
Chris Mason1bbc6212015-04-06 12:46:08 -070010711
10712 mutex_lock(&trans->transaction->cache_write_mutex);
10713 /*
Andrea Gelmini52042d82018-11-28 12:05:13 +010010714 * Make sure our free space cache IO is done before removing the
Chris Mason1bbc6212015-04-06 12:46:08 -070010715 * free space inode
10716 */
10717 spin_lock(&trans->transaction->dirty_bgs_lock);
10718 if (!list_empty(&block_group->io_list)) {
10719 list_del_init(&block_group->io_list);
10720
10721 WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode);
10722
10723 spin_unlock(&trans->transaction->dirty_bgs_lock);
Jeff Mahoneyafdb5712016-09-09 12:09:35 -040010724 btrfs_wait_cache_io(trans, block_group, path);
Chris Mason1bbc6212015-04-06 12:46:08 -070010725 btrfs_put_block_group(block_group);
10726 spin_lock(&trans->transaction->dirty_bgs_lock);
10727 }
10728
10729 if (!list_empty(&block_group->dirty_list)) {
10730 list_del_init(&block_group->dirty_list);
Josef Bacikba2c4d42018-12-03 10:20:33 -050010731 remove_rsv = true;
Chris Mason1bbc6212015-04-06 12:46:08 -070010732 btrfs_put_block_group(block_group);
10733 }
10734 spin_unlock(&trans->transaction->dirty_bgs_lock);
10735 mutex_unlock(&trans->transaction->cache_write_mutex);
10736
Josef Bacik0af3d002010-06-21 14:48:16 -040010737 if (!IS_ERR(inode)) {
Nikolay Borisov73f2e542017-02-20 13:50:59 +020010738 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
Jeff Mahoney79787ea2012-03-12 16:03:00 +010010739 if (ret) {
10740 btrfs_add_delayed_iput(inode);
10741 goto out;
10742 }
Josef Bacik0af3d002010-06-21 14:48:16 -040010743 clear_nlink(inode);
10744 /* One for the block groups ref */
10745 spin_lock(&block_group->lock);
10746 if (block_group->iref) {
10747 block_group->iref = 0;
10748 block_group->inode = NULL;
10749 spin_unlock(&block_group->lock);
10750 iput(inode);
10751 } else {
10752 spin_unlock(&block_group->lock);
10753 }
10754 /* One for our lookup ref */
Josef Bacik455757c2011-09-19 12:26:24 -040010755 btrfs_add_delayed_iput(inode);
Josef Bacik0af3d002010-06-21 14:48:16 -040010756 }
10757
10758 key.objectid = BTRFS_FREE_SPACE_OBJECTID;
10759 key.offset = block_group->key.objectid;
10760 key.type = 0;
10761
10762 ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
10763 if (ret < 0)
10764 goto out;
10765 if (ret > 0)
David Sterbab3b4aa72011-04-21 01:20:15 +020010766 btrfs_release_path(path);
Josef Bacik0af3d002010-06-21 14:48:16 -040010767 if (ret == 0) {
10768 ret = btrfs_del_item(trans, tree_root, path);
10769 if (ret)
10770 goto out;
David Sterbab3b4aa72011-04-21 01:20:15 +020010771 btrfs_release_path(path);
Josef Bacik0af3d002010-06-21 14:48:16 -040010772 }
10773
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010774 spin_lock(&fs_info->block_group_cache_lock);
Zheng Yan1a40e232008-09-26 10:09:34 -040010775 rb_erase(&block_group->cache_node,
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010776 &fs_info->block_group_cache_tree);
Filipe Manana292cbd52014-11-26 15:28:50 +000010777 RB_CLEAR_NODE(&block_group->cache_node);
Liu Boa1897fd2012-12-27 09:01:23 +000010778
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010779 if (fs_info->first_logical_byte == block_group->key.objectid)
10780 fs_info->first_logical_byte = (u64)-1;
10781 spin_unlock(&fs_info->block_group_cache_lock);
Josef Bacik817d52f2009-07-13 21:29:25 -040010782
Josef Bacik80eb2342008-10-29 14:49:05 -040010783 down_write(&block_group->space_info->groups_sem);
Chris Mason44fb5512009-06-04 15:34:51 -040010784 /*
10785 * we must use list_del_init so people can check to see if they
10786 * are still on the list after taking the semaphore
10787 */
10788 list_del_init(&block_group->list);
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040010789 if (list_empty(&block_group->space_info->block_groups[index])) {
Jeff Mahoneyc1895442014-05-27 12:59:57 -040010790 kobj = block_group->space_info->block_group_kobjs[index];
10791 block_group->space_info->block_group_kobjs[index] = NULL;
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010792 clear_avail_alloc_bits(fs_info, block_group->flags);
Jeff Mahoney6ab0a202013-11-01 13:07:04 -040010793 }
Josef Bacik80eb2342008-10-29 14:49:05 -040010794 up_write(&block_group->space_info->groups_sem);
Jeff Mahoneyc1895442014-05-27 12:59:57 -040010795 if (kobj) {
10796 kobject_del(kobj);
10797 kobject_put(kobj);
10798 }
Zheng Yan1a40e232008-09-26 10:09:34 -040010799
Filipe Manana4f69cb92014-11-26 15:28:51 +000010800 if (block_group->has_caching_ctl)
10801 caching_ctl = get_caching_control(block_group);
Josef Bacik817d52f2009-07-13 21:29:25 -040010802 if (block_group->cached == BTRFS_CACHE_STARTED)
Yan Zheng11833d62009-09-11 16:11:19 -040010803 wait_block_group_cache_done(block_group);
Filipe Manana4f69cb92014-11-26 15:28:51 +000010804 if (block_group->has_caching_ctl) {
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010805 down_write(&fs_info->commit_root_sem);
Filipe Manana4f69cb92014-11-26 15:28:51 +000010806 if (!caching_ctl) {
10807 struct btrfs_caching_control *ctl;
10808
10809 list_for_each_entry(ctl,
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010810 &fs_info->caching_block_groups, list)
Filipe Manana4f69cb92014-11-26 15:28:51 +000010811 if (ctl->block_group == block_group) {
10812 caching_ctl = ctl;
Elena Reshetova1e4f4712017-03-03 10:55:14 +020010813 refcount_inc(&caching_ctl->count);
Filipe Manana4f69cb92014-11-26 15:28:51 +000010814 break;
10815 }
10816 }
10817 if (caching_ctl)
10818 list_del_init(&caching_ctl->list);
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010819 up_write(&fs_info->commit_root_sem);
Filipe Manana4f69cb92014-11-26 15:28:51 +000010820 if (caching_ctl) {
10821 /* Once for the caching bgs list and once for us. */
10822 put_caching_control(caching_ctl);
10823 put_caching_control(caching_ctl);
10824 }
10825 }
Josef Bacik817d52f2009-07-13 21:29:25 -040010826
Josef Bacikce93ec52014-11-17 15:45:48 -050010827 spin_lock(&trans->transaction->dirty_bgs_lock);
Nikolay Borisov9a0ec832019-01-30 16:50:49 +020010828 WARN_ON(!list_empty(&block_group->dirty_list));
10829 WARN_ON(!list_empty(&block_group->io_list));
Josef Bacikce93ec52014-11-17 15:45:48 -050010830 spin_unlock(&trans->transaction->dirty_bgs_lock);
Nikolay Borisov9a0ec832019-01-30 16:50:49 +020010831
Josef Bacik817d52f2009-07-13 21:29:25 -040010832 btrfs_remove_free_space_cache(block_group);
10833
Yan Zhengc146afa2008-11-12 14:34:12 -050010834 spin_lock(&block_group->space_info->lock);
Filipe Manana75c68e92015-01-16 13:24:40 +000010835 list_del_init(&block_group->ro_list);
Zhao Lei18d018a2015-02-24 20:07:44 +080010836
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010837 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
Zhao Lei18d018a2015-02-24 20:07:44 +080010838 WARN_ON(block_group->space_info->total_bytes
10839 < block_group->key.offset);
10840 WARN_ON(block_group->space_info->bytes_readonly
10841 < block_group->key.offset);
10842 WARN_ON(block_group->space_info->disk_total
10843 < block_group->key.offset * factor);
10844 }
Yan Zhengc146afa2008-11-12 14:34:12 -050010845 block_group->space_info->total_bytes -= block_group->key.offset;
10846 block_group->space_info->bytes_readonly -= block_group->key.offset;
Josef Bacik89a55892010-10-14 14:52:27 -040010847 block_group->space_info->disk_total -= block_group->key.offset * factor;
Zhao Lei18d018a2015-02-24 20:07:44 +080010848
Yan Zhengc146afa2008-11-12 14:34:12 -050010849 spin_unlock(&block_group->space_info->lock);
Chris Mason283bb192009-07-24 16:30:55 -040010850
Josef Bacik0af3d002010-06-21 14:48:16 -040010851 memcpy(&key, &block_group->key, sizeof(key));
10852
David Sterba34441362016-10-04 19:34:27 +020010853 mutex_lock(&fs_info->chunk_mutex);
Filipe Manana04216822014-11-27 21:14:15 +000010854 spin_lock(&block_group->lock);
10855 block_group->removed = 1;
10856 /*
10857 * At this point trimming can't start on this block group, because we
10858 * removed the block group from the tree fs_info->block_group_cache_tree
10859 * so no one can't find it anymore and even if someone already got this
10860 * block group before we removed it from the rbtree, they have already
10861 * incremented block_group->trimming - if they didn't, they won't find
10862 * any free space entries because we already removed them all when we
10863 * called btrfs_remove_free_space_cache().
10864 *
10865 * And we must not remove the extent map from the fs_info->mapping_tree
10866 * to prevent the same logical address range and physical device space
10867 * ranges from being reused for a new block group. This is because our
10868 * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
10869 * completely transactionless, so while it is trimming a range the
10870 * currently running transaction might finish and a new one start,
10871 * allowing for new block groups to be created that can reuse the same
10872 * physical device locations unless we take this special care.
Jeff Mahoneye33e17e2015-06-15 09:41:19 -040010873 *
10874 * There may also be an implicit trim operation if the file system
10875 * is mounted with -odiscard. The same protections must remain
10876 * in place until the extents have been discarded completely when
10877 * the transaction commit has completed.
Filipe Manana04216822014-11-27 21:14:15 +000010878 */
10879 remove_em = (atomic_read(&block_group->trimming) == 0);
Filipe Manana04216822014-11-27 21:14:15 +000010880 spin_unlock(&block_group->lock);
Filipe Manana04216822014-11-27 21:14:15 +000010881
10882 if (remove_em) {
10883 struct extent_map_tree *em_tree;
10884
Jeff Mahoney0b246af2016-06-22 18:54:23 -040010885 em_tree = &fs_info->mapping_tree.map_tree;
Filipe Manana04216822014-11-27 21:14:15 +000010886 write_lock(&em_tree->lock);
10887 remove_extent_mapping(em_tree, em);
10888 write_unlock(&em_tree->lock);
10889 /* once for the tree */
10890 free_extent_map(em);
10891 }
10892
David Sterba34441362016-10-04 19:34:27 +020010893 mutex_unlock(&fs_info->chunk_mutex);
Filipe Manana8dbcd102014-12-02 18:07:49 +000010894
Nikolay Borisovf3f72772018-05-10 15:44:46 +030010895 ret = remove_block_group_free_space(trans, block_group);
Omar Sandoval1e144fb2015-09-29 20:50:37 -070010896 if (ret)
10897 goto out;
10898
Chris Masonfa9c0d792009-04-03 09:47:43 -040010899 btrfs_put_block_group(block_group);
10900 btrfs_put_block_group(block_group);
Zheng Yan1a40e232008-09-26 10:09:34 -040010901
10902 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
10903 if (ret > 0)
10904 ret = -EIO;
10905 if (ret < 0)
10906 goto out;
10907
10908 ret = btrfs_del_item(trans, root, path);
10909out:
Josef Bacikba2c4d42018-12-03 10:20:33 -050010910 if (remove_rsv)
10911 btrfs_delayed_refs_rsv_release(fs_info, 1);
Zheng Yan1a40e232008-09-26 10:09:34 -040010912 btrfs_free_path(path);
10913 return ret;
10914}
liuboacce9522011-01-06 19:30:25 +080010915
Filipe Manana8eab77f2015-11-13 23:57:16 +000010916struct btrfs_trans_handle *
Filipe Manana7fd01182015-11-13 23:57:17 +000010917btrfs_start_trans_remove_block_group(struct btrfs_fs_info *fs_info,
10918 const u64 chunk_offset)
Filipe Manana8eab77f2015-11-13 23:57:16 +000010919{
Filipe Manana7fd01182015-11-13 23:57:17 +000010920 struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree;
10921 struct extent_map *em;
10922 struct map_lookup *map;
10923 unsigned int num_items;
10924
10925 read_lock(&em_tree->lock);
10926 em = lookup_extent_mapping(em_tree, chunk_offset, 1);
10927 read_unlock(&em_tree->lock);
10928 ASSERT(em && em->start == chunk_offset);
10929
Filipe Manana8eab77f2015-11-13 23:57:16 +000010930 /*
Filipe Manana7fd01182015-11-13 23:57:17 +000010931 * We need to reserve 3 + N units from the metadata space info in order
10932 * to remove a block group (done at btrfs_remove_chunk() and at
10933 * btrfs_remove_block_group()), which are used for:
10934 *
Filipe Manana8eab77f2015-11-13 23:57:16 +000010935 * 1 unit for adding the free space inode's orphan (located in the tree
10936 * of tree roots).
Filipe Manana7fd01182015-11-13 23:57:17 +000010937 * 1 unit for deleting the block group item (located in the extent
10938 * tree).
10939 * 1 unit for deleting the free space item (located in tree of tree
10940 * roots).
10941 * N units for deleting N device extent items corresponding to each
10942 * stripe (located in the device tree).
10943 *
10944 * In order to remove a block group we also need to reserve units in the
10945 * system space info in order to update the chunk tree (update one or
10946 * more device items and remove one chunk item), but this is done at
10947 * btrfs_remove_chunk() through a call to check_system_chunk().
Filipe Manana8eab77f2015-11-13 23:57:16 +000010948 */
Jeff Mahoney95617d62015-06-03 10:55:48 -040010949 map = em->map_lookup;
Filipe Manana7fd01182015-11-13 23:57:17 +000010950 num_items = 3 + map->num_stripes;
10951 free_extent_map(em);
10952
Filipe Manana8eab77f2015-11-13 23:57:16 +000010953 return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root,
Filipe Manana7fd01182015-11-13 23:57:17 +000010954 num_items, 1);
Filipe Manana8eab77f2015-11-13 23:57:16 +000010955}
10956
Josef Bacik47ab2a62014-09-18 11:20:02 -040010957/*
10958 * Process the unused_bgs list and remove any that don't have any allocated
10959 * space inside of them.
10960 */
10961void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
10962{
10963 struct btrfs_block_group_cache *block_group;
10964 struct btrfs_space_info *space_info;
Josef Bacik47ab2a62014-09-18 11:20:02 -040010965 struct btrfs_trans_handle *trans;
10966 int ret = 0;
10967
Josef Bacikafcdd122016-09-02 15:40:02 -040010968 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
Josef Bacik47ab2a62014-09-18 11:20:02 -040010969 return;
10970
10971 spin_lock(&fs_info->unused_bgs_lock);
10972 while (!list_empty(&fs_info->unused_bgs)) {
10973 u64 start, end;
Jeff Mahoneye33e17e2015-06-15 09:41:19 -040010974 int trimming;
Josef Bacik47ab2a62014-09-18 11:20:02 -040010975
10976 block_group = list_first_entry(&fs_info->unused_bgs,
10977 struct btrfs_block_group_cache,
10978 bg_list);
Josef Bacik47ab2a62014-09-18 11:20:02 -040010979 list_del_init(&block_group->bg_list);
Zhao Leiaefbe9a2015-09-29 21:03:54 +080010980
10981 space_info = block_group->space_info;
10982
Josef Bacik47ab2a62014-09-18 11:20:02 -040010983 if (ret || btrfs_mixed_space_info(space_info)) {
10984 btrfs_put_block_group(block_group);
10985 continue;
10986 }
10987 spin_unlock(&fs_info->unused_bgs_lock);
10988
Zhao Leid5f2e332015-10-08 18:46:44 +080010989 mutex_lock(&fs_info->delete_unused_bgs_mutex);
Filipe Manana67c5e7d2015-06-11 00:58:53 +010010990
Josef Bacik47ab2a62014-09-18 11:20:02 -040010991 /* Don't want to race with allocators so take the groups_sem */
10992 down_write(&space_info->groups_sem);
10993 spin_lock(&block_group->lock);
Qu Wenruo43794442018-06-22 12:35:00 +080010994 if (block_group->reserved || block_group->pinned ||
Josef Bacik47ab2a62014-09-18 11:20:02 -040010995 btrfs_block_group_used(&block_group->item) ||
Chris Mason19c4d2f2016-10-10 13:43:31 -070010996 block_group->ro ||
Zhao Leiaefbe9a2015-09-29 21:03:54 +080010997 list_is_singular(&block_group->list)) {
Josef Bacik47ab2a62014-09-18 11:20:02 -040010998 /*
10999 * We want to bail if we made new allocations or have
11000 * outstanding allocations in this block group. We do
11001 * the ro check in case balance is currently acting on
11002 * this block group.
11003 */
Qu Wenruo4ed0a7a2018-04-26 17:17:20 +080011004 trace_btrfs_skip_unused_block_group(block_group);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011005 spin_unlock(&block_group->lock);
11006 up_write(&space_info->groups_sem);
11007 goto next;
11008 }
11009 spin_unlock(&block_group->lock);
11010
11011 /* We don't want to force the issue, only flip if it's ok. */
Zhaolei868f4012015-08-05 16:43:27 +080011012 ret = inc_block_group_ro(block_group, 0);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011013 up_write(&space_info->groups_sem);
11014 if (ret < 0) {
11015 ret = 0;
11016 goto next;
11017 }
11018
11019 /*
11020 * Want to do this before we do anything else so we can recover
11021 * properly if we fail to join the transaction.
11022 */
Filipe Manana7fd01182015-11-13 23:57:17 +000011023 trans = btrfs_start_trans_remove_block_group(fs_info,
11024 block_group->key.objectid);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011025 if (IS_ERR(trans)) {
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040011026 btrfs_dec_block_group_ro(block_group);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011027 ret = PTR_ERR(trans);
11028 goto next;
11029 }
11030
11031 /*
11032 * We could have pending pinned extents for this block group,
11033 * just delete them, we don't care about them anymore.
11034 */
11035 start = block_group->key.objectid;
11036 end = start + block_group->key.offset - 1;
Filipe Mananad4b450c2015-01-29 19:18:25 +000011037 /*
11038 * Hold the unused_bg_unpin_mutex lock to avoid racing with
11039 * btrfs_finish_extent_commit(). If we are at transaction N,
11040 * another task might be running finish_extent_commit() for the
11041 * previous transaction N - 1, and have seen a range belonging
11042 * to the block group in freed_extents[] before we were able to
11043 * clear the whole block group range from freed_extents[]. This
11044 * means that task can lookup for the block group after we
11045 * unpinned it from freed_extents[] and removed it, leading to
11046 * a BUG_ON() at btrfs_unpin_extent_range().
11047 */
11048 mutex_lock(&fs_info->unused_bg_unpin_mutex);
Filipe Manana758eb512014-11-03 14:08:39 +000011049 ret = clear_extent_bits(&fs_info->freed_extents[0], start, end,
David Sterba91166212016-04-26 23:54:39 +020011050 EXTENT_DIRTY);
Filipe Manana758eb512014-11-03 14:08:39 +000011051 if (ret) {
Filipe Mananad4b450c2015-01-29 19:18:25 +000011052 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040011053 btrfs_dec_block_group_ro(block_group);
Filipe Manana758eb512014-11-03 14:08:39 +000011054 goto end_trans;
11055 }
11056 ret = clear_extent_bits(&fs_info->freed_extents[1], start, end,
David Sterba91166212016-04-26 23:54:39 +020011057 EXTENT_DIRTY);
Filipe Manana758eb512014-11-03 14:08:39 +000011058 if (ret) {
Filipe Mananad4b450c2015-01-29 19:18:25 +000011059 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040011060 btrfs_dec_block_group_ro(block_group);
Filipe Manana758eb512014-11-03 14:08:39 +000011061 goto end_trans;
11062 }
Filipe Mananad4b450c2015-01-29 19:18:25 +000011063 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011064
11065 /* Reset pinned so btrfs_put_block_group doesn't complain */
Zhao Leic30666d2015-02-25 14:17:20 +080011066 spin_lock(&space_info->lock);
11067 spin_lock(&block_group->lock);
11068
Lu Fengqie2907c12018-10-24 20:24:02 +080011069 update_bytes_pinned(space_info, -block_group->pinned);
Zhao Leic30666d2015-02-25 14:17:20 +080011070 space_info->bytes_readonly += block_group->pinned;
Ethan Liendec59fa2018-07-13 16:50:42 +080011071 percpu_counter_add_batch(&space_info->total_bytes_pinned,
11072 -block_group->pinned,
11073 BTRFS_TOTAL_BYTES_PINNED_BATCH);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011074 block_group->pinned = 0;
11075
Zhao Leic30666d2015-02-25 14:17:20 +080011076 spin_unlock(&block_group->lock);
11077 spin_unlock(&space_info->lock);
11078
Jeff Mahoneye33e17e2015-06-15 09:41:19 -040011079 /* DISCARD can flip during remount */
Jeff Mahoney0b246af2016-06-22 18:54:23 -040011080 trimming = btrfs_test_opt(fs_info, DISCARD);
Jeff Mahoneye33e17e2015-06-15 09:41:19 -040011081
11082 /* Implicit trim during transaction commit. */
11083 if (trimming)
11084 btrfs_get_block_group_trimming(block_group);
11085
Josef Bacik47ab2a62014-09-18 11:20:02 -040011086 /*
11087 * Btrfs_remove_chunk will abort the transaction if things go
11088 * horribly wrong.
11089 */
Nikolay Borisov97aff912018-07-20 19:37:53 +030011090 ret = btrfs_remove_chunk(trans, block_group->key.objectid);
Jeff Mahoneye33e17e2015-06-15 09:41:19 -040011091
11092 if (ret) {
11093 if (trimming)
11094 btrfs_put_block_group_trimming(block_group);
11095 goto end_trans;
11096 }
11097
11098 /*
11099 * If we're not mounted with -odiscard, we can just forget
11100 * about this block group. Otherwise we'll need to wait
11101 * until transaction commit to do the actual discard.
11102 */
11103 if (trimming) {
Filipe Manana348a0012015-11-27 12:16:16 +000011104 spin_lock(&fs_info->unused_bgs_lock);
11105 /*
11106 * A concurrent scrub might have added us to the list
11107 * fs_info->unused_bgs, so use a list_move operation
11108 * to add the block group to the deleted_bgs list.
11109 */
Jeff Mahoneye33e17e2015-06-15 09:41:19 -040011110 list_move(&block_group->bg_list,
11111 &trans->transaction->deleted_bgs);
Filipe Manana348a0012015-11-27 12:16:16 +000011112 spin_unlock(&fs_info->unused_bgs_lock);
Jeff Mahoneye33e17e2015-06-15 09:41:19 -040011113 btrfs_get_block_group(block_group);
11114 }
Filipe Manana758eb512014-11-03 14:08:39 +000011115end_trans:
Jeff Mahoney3a45bb22016-09-09 21:39:03 -040011116 btrfs_end_transaction(trans);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011117next:
Zhao Leid5f2e332015-10-08 18:46:44 +080011118 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
Josef Bacik47ab2a62014-09-18 11:20:02 -040011119 btrfs_put_block_group(block_group);
11120 spin_lock(&fs_info->unused_bgs_lock);
11121 }
11122 spin_unlock(&fs_info->unused_bgs_lock);
11123}
11124
liuboc59021f2011-03-07 02:13:14 +000011125int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
11126{
liubo1aba86d2011-04-08 08:44:37 +000011127 struct btrfs_super_block *disk_super;
11128 u64 features;
11129 u64 flags;
11130 int mixed = 0;
liuboc59021f2011-03-07 02:13:14 +000011131 int ret;
11132
David Sterba6c417612011-04-13 15:41:04 +020011133 disk_super = fs_info->super_copy;
liubo1aba86d2011-04-08 08:44:37 +000011134 if (!btrfs_super_root(disk_super))
Dan Carpenter0dc924c52016-01-13 15:21:17 +030011135 return -EINVAL;
liuboc59021f2011-03-07 02:13:14 +000011136
liubo1aba86d2011-04-08 08:44:37 +000011137 features = btrfs_super_incompat_flags(disk_super);
11138 if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
11139 mixed = 1;
liuboc59021f2011-03-07 02:13:14 +000011140
liubo1aba86d2011-04-08 08:44:37 +000011141 flags = BTRFS_BLOCK_GROUP_SYSTEM;
Lu Fengqi4ca61682018-05-28 14:30:27 +080011142 ret = create_space_info(fs_info, flags);
liuboc59021f2011-03-07 02:13:14 +000011143 if (ret)
liubo1aba86d2011-04-08 08:44:37 +000011144 goto out;
liuboc59021f2011-03-07 02:13:14 +000011145
liubo1aba86d2011-04-08 08:44:37 +000011146 if (mixed) {
11147 flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
Lu Fengqi4ca61682018-05-28 14:30:27 +080011148 ret = create_space_info(fs_info, flags);
liubo1aba86d2011-04-08 08:44:37 +000011149 } else {
11150 flags = BTRFS_BLOCK_GROUP_METADATA;
Lu Fengqi4ca61682018-05-28 14:30:27 +080011151 ret = create_space_info(fs_info, flags);
liubo1aba86d2011-04-08 08:44:37 +000011152 if (ret)
11153 goto out;
11154
11155 flags = BTRFS_BLOCK_GROUP_DATA;
Lu Fengqi4ca61682018-05-28 14:30:27 +080011156 ret = create_space_info(fs_info, flags);
liubo1aba86d2011-04-08 08:44:37 +000011157 }
11158out:
liuboc59021f2011-03-07 02:13:14 +000011159 return ret;
11160}
11161
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040011162int btrfs_error_unpin_extent_range(struct btrfs_fs_info *fs_info,
11163 u64 start, u64 end)
liuboacce9522011-01-06 19:30:25 +080011164{
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040011165 return unpin_extent_range(fs_info, start, end, false);
liuboacce9522011-01-06 19:30:25 +080011166}
11167
Jeff Mahoney499f3772015-06-15 09:41:17 -040011168/*
11169 * It used to be that old block groups would be left around forever.
11170 * Iterating over them would be enough to trim unused space. Since we
11171 * now automatically remove them, we also need to iterate over unallocated
11172 * space.
11173 *
11174 * We don't want a transaction for this since the discard may take a
11175 * substantial amount of time. We don't require that a transaction be
11176 * running, but we do need to take a running transaction into account
Jeff Mahoneyfee7acc2018-09-06 17:18:16 -040011177 * to ensure that we're not discarding chunks that were released or
11178 * allocated in the current transaction.
Jeff Mahoney499f3772015-06-15 09:41:17 -040011179 *
11180 * Holding the chunks lock will prevent other threads from allocating
11181 * or releasing chunks, but it won't prevent a running transaction
11182 * from committing and releasing the memory that the pending chunks
11183 * list head uses. For that, we need to take a reference to the
Jeff Mahoneyfee7acc2018-09-06 17:18:16 -040011184 * transaction and hold the commit root sem. We only need to hold
11185 * it while performing the free space search since we have already
11186 * held back allocations.
Jeff Mahoney499f3772015-06-15 09:41:17 -040011187 */
11188static int btrfs_trim_free_extents(struct btrfs_device *device,
Nikolay Borisovc2d1b3a2019-03-25 14:31:21 +020011189 struct fstrim_range *range, u64 *trimmed)
Jeff Mahoney499f3772015-06-15 09:41:17 -040011190{
Nikolay Borisov929be172019-03-27 14:24:18 +020011191 u64 start, len = 0, end = 0;
Jeff Mahoney499f3772015-06-15 09:41:17 -040011192 int ret;
11193
Nikolay Borisov929be172019-03-27 14:24:18 +020011194 start = max_t(u64, range->start, SZ_1M);
Jeff Mahoney499f3772015-06-15 09:41:17 -040011195 *trimmed = 0;
11196
Jeff Mahoney0be88e32018-09-06 17:18:15 -040011197 /* Discard not supported = nothing to do. */
11198 if (!blk_queue_discard(bdev_get_queue(device->bdev)))
11199 return 0;
11200
Andrea Gelmini52042d82018-11-28 12:05:13 +010011201 /* Not writable = nothing to do. */
Anand Jainebbede42017-12-04 12:54:52 +080011202 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
Jeff Mahoney499f3772015-06-15 09:41:17 -040011203 return 0;
11204
11205 /* No free space = nothing to do. */
11206 if (device->total_bytes <= device->bytes_used)
11207 return 0;
11208
11209 ret = 0;
11210
11211 while (1) {
Jeff Mahoneyfb456252016-06-22 18:54:56 -040011212 struct btrfs_fs_info *fs_info = device->fs_info;
Jeff Mahoney499f3772015-06-15 09:41:17 -040011213 u64 bytes;
11214
11215 ret = mutex_lock_interruptible(&fs_info->chunk_mutex);
11216 if (ret)
Jeff Mahoneyfee7acc2018-09-06 17:18:16 -040011217 break;
Jeff Mahoney499f3772015-06-15 09:41:17 -040011218
Nikolay Borisov929be172019-03-27 14:24:18 +020011219 find_first_clear_extent_bit(&device->alloc_state, start,
11220 &start, &end,
11221 CHUNK_TRIMMED | CHUNK_ALLOCATED);
11222 /*
11223 * If find_first_clear_extent_bit find a range that spans the
11224 * end of the device it will set end to -1, in this case it's up
11225 * to the caller to trim the value to the size of the device.
11226 */
11227 end = min(end, device->total_bytes - 1);
11228 len = end - start + 1;
Jeff Mahoney499f3772015-06-15 09:41:17 -040011229
Nikolay Borisov929be172019-03-27 14:24:18 +020011230 /* We didn't find any extents */
11231 if (!len) {
Jeff Mahoney499f3772015-06-15 09:41:17 -040011232 mutex_unlock(&fs_info->chunk_mutex);
Nikolay Borisov929be172019-03-27 14:24:18 +020011233 ret = 0;
Jeff Mahoney499f3772015-06-15 09:41:17 -040011234 break;
11235 }
11236
Nikolay Borisov929be172019-03-27 14:24:18 +020011237 /* Keep going until we satisfy minlen or reach end of space */
11238 if (len < range->minlen) {
11239 mutex_unlock(&fs_info->chunk_mutex);
11240 start += len;
11241 continue;
11242 }
11243
Nikolay Borisovc2d1b3a2019-03-25 14:31:21 +020011244 /* If we are out of the passed range break */
11245 if (start > range->start + range->len - 1) {
11246 mutex_unlock(&fs_info->chunk_mutex);
Nikolay Borisovc2d1b3a2019-03-25 14:31:21 +020011247 break;
11248 }
11249
11250 start = max(range->start, start);
11251 len = min(range->len, len);
11252
Nikolay Borisov929be172019-03-27 14:24:18 +020011253 ret = btrfs_issue_discard(device->bdev, start, len,
11254 &bytes);
11255 if (!ret)
11256 set_extent_bits(&device->alloc_state, start,
11257 start + bytes - 1,
11258 CHUNK_TRIMMED);
Jeff Mahoney499f3772015-06-15 09:41:17 -040011259 mutex_unlock(&fs_info->chunk_mutex);
11260
11261 if (ret)
11262 break;
11263
11264 start += len;
11265 *trimmed += bytes;
11266
Nikolay Borisovc2d1b3a2019-03-25 14:31:21 +020011267 /* We've trimmed enough */
11268 if (*trimmed >= range->len)
11269 break;
11270
Jeff Mahoney499f3772015-06-15 09:41:17 -040011271 if (fatal_signal_pending(current)) {
11272 ret = -ERESTARTSYS;
11273 break;
11274 }
11275
11276 cond_resched();
11277 }
11278
11279 return ret;
11280}
11281
Qu Wenruo93bba242018-09-07 14:16:23 +080011282/*
11283 * Trim the whole filesystem by:
11284 * 1) trimming the free space in each block group
11285 * 2) trimming the unallocated space on each device
11286 *
11287 * This will also continue trimming even if a block group or device encounters
11288 * an error. The return value will be the last error, or 0 if nothing bad
11289 * happens.
11290 */
Jeff Mahoney2ff7e612016-06-22 18:54:24 -040011291int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range)
Li Dongyangf7039b12011-03-24 10:24:28 +000011292{
Li Dongyangf7039b12011-03-24 10:24:28 +000011293 struct btrfs_block_group_cache *cache = NULL;
Jeff Mahoney499f3772015-06-15 09:41:17 -040011294 struct btrfs_device *device;
11295 struct list_head *devices;
Li Dongyangf7039b12011-03-24 10:24:28 +000011296 u64 group_trimmed;
11297 u64 start;
11298 u64 end;
11299 u64 trimmed = 0;
Qu Wenruo93bba242018-09-07 14:16:23 +080011300 u64 bg_failed = 0;
11301 u64 dev_failed = 0;
11302 int bg_ret = 0;
11303 int dev_ret = 0;
Li Dongyangf7039b12011-03-24 10:24:28 +000011304 int ret = 0;
11305
Qu Wenruo6ba9fc82018-09-07 14:16:24 +080011306 cache = btrfs_lookup_first_block_group(fs_info, range->start);
David Sterbaf87b7eb2019-03-20 12:01:07 +010011307 for (; cache; cache = next_block_group(cache)) {
Li Dongyangf7039b12011-03-24 10:24:28 +000011308 if (cache->key.objectid >= (range->start + range->len)) {
11309 btrfs_put_block_group(cache);
11310 break;
11311 }
11312
11313 start = max(range->start, cache->key.objectid);
11314 end = min(range->start + range->len,
11315 cache->key.objectid + cache->key.offset);
11316
11317 if (end - start >= range->minlen) {
11318 if (!block_group_cache_done(cache)) {
Liu Bof6373bf2012-12-27 09:01:18 +000011319 ret = cache_block_group(cache, 0);
Josef Bacik1be41b72013-06-12 13:56:06 -040011320 if (ret) {
Qu Wenruo93bba242018-09-07 14:16:23 +080011321 bg_failed++;
11322 bg_ret = ret;
11323 continue;
Josef Bacik1be41b72013-06-12 13:56:06 -040011324 }
11325 ret = wait_block_group_cache_done(cache);
11326 if (ret) {
Qu Wenruo93bba242018-09-07 14:16:23 +080011327 bg_failed++;
11328 bg_ret = ret;
11329 continue;
Josef Bacik1be41b72013-06-12 13:56:06 -040011330 }
Li Dongyangf7039b12011-03-24 10:24:28 +000011331 }
11332 ret = btrfs_trim_block_group(cache,
11333 &group_trimmed,
11334 start,
11335 end,
11336 range->minlen);
11337
11338 trimmed += group_trimmed;
11339 if (ret) {
Qu Wenruo93bba242018-09-07 14:16:23 +080011340 bg_failed++;
11341 bg_ret = ret;
11342 continue;
Li Dongyangf7039b12011-03-24 10:24:28 +000011343 }
11344 }
Li Dongyangf7039b12011-03-24 10:24:28 +000011345 }
11346
Qu Wenruo93bba242018-09-07 14:16:23 +080011347 if (bg_failed)
11348 btrfs_warn(fs_info,
11349 "failed to trim %llu block group(s), last error %d",
11350 bg_failed, bg_ret);
Jeff Mahoney0b246af2016-06-22 18:54:23 -040011351 mutex_lock(&fs_info->fs_devices->device_list_mutex);
Jeff Mahoneyd4e329d2018-09-06 17:18:14 -040011352 devices = &fs_info->fs_devices->devices;
11353 list_for_each_entry(device, devices, dev_list) {
Nikolay Borisovc2d1b3a2019-03-25 14:31:21 +020011354 ret = btrfs_trim_free_extents(device, range, &group_trimmed);
Qu Wenruo93bba242018-09-07 14:16:23 +080011355 if (ret) {
11356 dev_failed++;
11357 dev_ret = ret;
Jeff Mahoney499f3772015-06-15 09:41:17 -040011358 break;
Qu Wenruo93bba242018-09-07 14:16:23 +080011359 }
Jeff Mahoney499f3772015-06-15 09:41:17 -040011360
11361 trimmed += group_trimmed;
11362 }
Jeff Mahoney0b246af2016-06-22 18:54:23 -040011363 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
Jeff Mahoney499f3772015-06-15 09:41:17 -040011364
Qu Wenruo93bba242018-09-07 14:16:23 +080011365 if (dev_failed)
11366 btrfs_warn(fs_info,
11367 "failed to trim %llu device(s), last error %d",
11368 dev_failed, dev_ret);
Li Dongyangf7039b12011-03-24 10:24:28 +000011369 range->len = trimmed;
Qu Wenruo93bba242018-09-07 14:16:23 +080011370 if (bg_ret)
11371 return bg_ret;
11372 return dev_ret;
Li Dongyangf7039b12011-03-24 10:24:28 +000011373}
Miao Xie8257b2d2014-03-06 13:38:19 +080011374
11375/*
David Sterbaea14b57f2017-06-22 02:19:11 +020011376 * btrfs_{start,end}_write_no_snapshotting() are similar to
Filipe Manana9ea24bb2014-10-29 11:57:59 +000011377 * mnt_{want,drop}_write(), they are used to prevent some tasks from writing
11378 * data into the page cache through nocow before the subvolume is snapshoted,
11379 * but flush the data into disk after the snapshot creation, or to prevent
David Sterbaea14b57f2017-06-22 02:19:11 +020011380 * operations while snapshotting is ongoing and that cause the snapshot to be
Filipe Manana9ea24bb2014-10-29 11:57:59 +000011381 * inconsistent (writes followed by expanding truncates for example).
Miao Xie8257b2d2014-03-06 13:38:19 +080011382 */
David Sterbaea14b57f2017-06-22 02:19:11 +020011383void btrfs_end_write_no_snapshotting(struct btrfs_root *root)
Miao Xie8257b2d2014-03-06 13:38:19 +080011384{
11385 percpu_counter_dec(&root->subv_writers->counter);
David Sterba093258e2018-02-26 16:15:17 +010011386 cond_wake_up(&root->subv_writers->wait);
Miao Xie8257b2d2014-03-06 13:38:19 +080011387}
11388
David Sterbaea14b57f2017-06-22 02:19:11 +020011389int btrfs_start_write_no_snapshotting(struct btrfs_root *root)
Miao Xie8257b2d2014-03-06 13:38:19 +080011390{
David Sterbaea14b57f2017-06-22 02:19:11 +020011391 if (atomic_read(&root->will_be_snapshotted))
Miao Xie8257b2d2014-03-06 13:38:19 +080011392 return 0;
11393
11394 percpu_counter_inc(&root->subv_writers->counter);
11395 /*
11396 * Make sure counter is updated before we check for snapshot creation.
11397 */
11398 smp_mb();
David Sterbaea14b57f2017-06-22 02:19:11 +020011399 if (atomic_read(&root->will_be_snapshotted)) {
11400 btrfs_end_write_no_snapshotting(root);
Miao Xie8257b2d2014-03-06 13:38:19 +080011401 return 0;
11402 }
11403 return 1;
11404}
Zhao Lei0bc19f902016-01-06 18:56:36 +080011405
Zhao Lei0bc19f902016-01-06 18:56:36 +080011406void btrfs_wait_for_snapshot_creation(struct btrfs_root *root)
11407{
11408 while (true) {
11409 int ret;
11410
David Sterbaea14b57f2017-06-22 02:19:11 +020011411 ret = btrfs_start_write_no_snapshotting(root);
Zhao Lei0bc19f902016-01-06 18:56:36 +080011412 if (ret)
11413 break;
Peter Zijlstra46259562018-03-15 11:43:08 +010011414 wait_var_event(&root->will_be_snapshotted,
11415 !atomic_read(&root->will_be_snapshotted));
Zhao Lei0bc19f902016-01-06 18:56:36 +080011416 }
11417}
Qu Wenruo031f24d2018-05-22 16:43:47 +080011418
11419void btrfs_mark_bg_unused(struct btrfs_block_group_cache *bg)
11420{
11421 struct btrfs_fs_info *fs_info = bg->fs_info;
11422
11423 spin_lock(&fs_info->unused_bgs_lock);
11424 if (list_empty(&bg->bg_list)) {
11425 btrfs_get_block_group(bg);
11426 trace_btrfs_add_unused_block_group(bg);
11427 list_add_tail(&bg->bg_list, &fs_info->unused_bgs);
11428 }
11429 spin_unlock(&fs_info->unused_bgs_lock);
11430}