blob: 6e87178d34a2fd865979eacad276cead84781f0a [file] [log] [blame]
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +09001/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002 * fs/f2fs/node.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/mpage.h>
14#include <linux/backing-dev.h>
15#include <linux/blkdev.h>
16#include <linux/pagevec.h>
17#include <linux/swap.h>
18
19#include "f2fs.h"
20#include "node.h"
21#include "segment.h"
Jaegeuk Kim9e4ded32014-12-17 19:58:58 -080022#include "trace.h"
Namjae Jeon51dd6242013-04-20 01:28:52 +090023#include <trace/events/f2fs.h>
Jaegeuk Kime05df3b2012-11-02 17:08:50 +090024
Gu Zhengf978f5a2014-02-21 18:08:29 +080025#define on_build_free_nids(nmi) mutex_is_locked(&nm_i->build_lock)
26
Jaegeuk Kime05df3b2012-11-02 17:08:50 +090027static struct kmem_cache *nat_entry_slab;
28static struct kmem_cache *free_nid_slab;
Chao Yuaec71382014-06-24 09:18:20 +080029static struct kmem_cache *nat_entry_set_slab;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +090030
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +090031bool available_free_memory(struct f2fs_sb_info *sbi, int type)
Jaegeuk Kimcdfc41c2014-03-19 13:31:37 +090032{
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +090033 struct f2fs_nm_info *nm_i = NM_I(sbi);
Jaegeuk Kimcdfc41c2014-03-19 13:31:37 +090034 struct sysinfo val;
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080035 unsigned long avail_ram;
Jaegeuk Kimcdfc41c2014-03-19 13:31:37 +090036 unsigned long mem_size = 0;
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +090037 bool res = false;
Jaegeuk Kimcdfc41c2014-03-19 13:31:37 +090038
39 si_meminfo(&val);
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080040
41 /* only uses low memory */
42 avail_ram = val.totalram - val.totalhigh;
43
Chao Yu429511c2015-02-05 17:54:31 +080044 /*
45 * give 25%, 25%, 50%, 50%, 50% memory for each components respectively
46 */
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +090047 if (type == FREE_NIDS) {
Chao Yub8559dc2016-10-12 19:28:29 +080048 mem_size = (nm_i->nid_cnt[FREE_NID_LIST] *
49 sizeof(struct free_nid)) >> PAGE_SHIFT;
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080050 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +090051 } else if (type == NAT_ENTRIES) {
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080052 mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >>
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +030053 PAGE_SHIFT;
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080054 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
Jaegeuk Kime589c2c2016-06-02 15:24:24 -070055 if (excess_cached_nats(sbi))
56 res = false;
Jaegeuk Kima1257022015-10-08 10:40:07 -070057 } else if (type == DIRTY_DENTS) {
58 if (sbi->sb->s_bdi->wb.dirty_exceeded)
59 return false;
60 mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);
61 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080062 } else if (type == INO_ENTRIES) {
63 int i;
64
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080065 for (i = 0; i <= UPDATE_INO; i++)
Kinglong Mee8f73cbb2017-03-18 09:26:13 +080066 mem_size += sbi->im[i].ino_num *
67 sizeof(struct ino_entry);
68 mem_size >>= PAGE_SHIFT;
Jaegeuk Kime5e7ea32014-11-06 15:24:46 -080069 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
Chao Yu429511c2015-02-05 17:54:31 +080070 } else if (type == EXTENT_CACHE) {
Jaegeuk Kim7441cce2015-12-21 19:20:15 -080071 mem_size = (atomic_read(&sbi->total_ext_tree) *
72 sizeof(struct extent_tree) +
Chao Yu429511c2015-02-05 17:54:31 +080073 atomic_read(&sbi->total_ext_node) *
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +030074 sizeof(struct extent_node)) >> PAGE_SHIFT;
Chao Yu429511c2015-02-05 17:54:31 +080075 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
Jaegeuk Kim1e843712014-12-09 06:08:59 -080076 } else {
Jaegeuk Kim1663cae2016-01-09 16:14:08 -080077 if (!sbi->sb->s_bdi->wb.dirty_exceeded)
78 return true;
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +090079 }
80 return res;
Jaegeuk Kimcdfc41c2014-03-19 13:31:37 +090081}
82
Jaegeuk Kime05df3b2012-11-02 17:08:50 +090083static void clear_node_page_dirty(struct page *page)
84{
85 struct address_space *mapping = page->mapping;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +090086 unsigned int long flags;
87
88 if (PageDirty(page)) {
89 spin_lock_irqsave(&mapping->tree_lock, flags);
90 radix_tree_tag_clear(&mapping->page_tree,
91 page_index(page),
92 PAGECACHE_TAG_DIRTY);
93 spin_unlock_irqrestore(&mapping->tree_lock, flags);
94
95 clear_page_dirty_for_io(page);
Jaegeuk Kim40813632014-09-02 15:31:18 -070096 dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +090097 }
98 ClearPageUptodate(page);
99}
100
101static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
102{
103 pgoff_t index = current_nat_addr(sbi, nid);
104 return get_meta_page(sbi, index);
105}
106
107static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
108{
109 struct page *src_page;
110 struct page *dst_page;
111 pgoff_t src_off;
112 pgoff_t dst_off;
113 void *src_addr;
114 void *dst_addr;
115 struct f2fs_nm_info *nm_i = NM_I(sbi);
116
117 src_off = current_nat_addr(sbi, nid);
118 dst_off = next_nat_addr(sbi, src_off);
119
120 /* get current nat block page with lock */
121 src_page = get_meta_page(sbi, src_off);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900122 dst_page = grab_meta_page(sbi, dst_off);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700123 f2fs_bug_on(sbi, PageDirty(src_page));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900124
125 src_addr = page_address(src_page);
126 dst_addr = page_address(dst_page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300127 memcpy(dst_addr, src_addr, PAGE_SIZE);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900128 set_page_dirty(dst_page);
129 f2fs_put_page(src_page, 1);
130
131 set_to_next_nat(nm_i, nid);
132
133 return dst_page;
134}
135
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900136static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n)
137{
138 return radix_tree_lookup(&nm_i->nat_root, n);
139}
140
141static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,
142 nid_t start, unsigned int nr, struct nat_entry **ep)
143{
144 return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);
145}
146
147static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)
148{
149 list_del(&e->list);
150 radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));
151 nm_i->nat_cnt--;
152 kmem_cache_free(nat_entry_slab, e);
153}
154
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -0700155static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,
156 struct nat_entry *ne)
157{
158 nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid);
159 struct nat_entry_set *head;
160
161 if (get_nat_flag(ne, IS_DIRTY))
162 return;
Jaegeuk Kim9be32d72014-12-05 10:39:49 -0800163
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -0700164 head = radix_tree_lookup(&nm_i->nat_set_root, set);
165 if (!head) {
Jaegeuk Kim80c54502015-08-20 08:51:56 -0700166 head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_NOFS);
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -0700167
168 INIT_LIST_HEAD(&head->entry_list);
169 INIT_LIST_HEAD(&head->set_list);
170 head->set = set;
171 head->entry_cnt = 0;
Jaegeuk Kim9be32d72014-12-05 10:39:49 -0800172 f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head);
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -0700173 }
174 list_move_tail(&ne->list, &head->entry_list);
175 nm_i->dirty_nat_cnt++;
176 head->entry_cnt++;
177 set_nat_flag(ne, IS_DIRTY, true);
178}
179
180static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i,
Kinglong Mee0b28b712017-02-28 21:34:47 +0800181 struct nat_entry_set *set, struct nat_entry *ne)
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -0700182{
Kinglong Mee0b28b712017-02-28 21:34:47 +0800183 list_move_tail(&ne->list, &nm_i->nat_entries);
184 set_nat_flag(ne, IS_DIRTY, false);
185 set->entry_cnt--;
186 nm_i->dirty_nat_cnt--;
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -0700187}
188
189static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,
190 nid_t start, unsigned int nr, struct nat_entry_set **ep)
191{
192 return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep,
193 start, nr);
194}
195
Jaegeuk Kim2dcf51a2015-04-29 18:31:19 -0700196int need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid)
197{
198 struct f2fs_nm_info *nm_i = NM_I(sbi);
199 struct nat_entry *e;
200 bool need = false;
201
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700202 down_read(&nm_i->nat_tree_lock);
Jaegeuk Kim2dcf51a2015-04-29 18:31:19 -0700203 e = __lookup_nat_cache(nm_i, nid);
204 if (e) {
205 if (!get_nat_flag(e, IS_CHECKPOINTED) &&
206 !get_nat_flag(e, HAS_FSYNCED_INODE))
207 need = true;
208 }
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700209 up_read(&nm_i->nat_tree_lock);
Jaegeuk Kim2dcf51a2015-04-29 18:31:19 -0700210 return need;
211}
212
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700213bool is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900214{
215 struct f2fs_nm_info *nm_i = NM_I(sbi);
216 struct nat_entry *e;
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700217 bool is_cp = true;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900218
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700219 down_read(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900220 e = __lookup_nat_cache(nm_i, nid);
Jaegeuk Kim7ef35e32014-09-15 12:07:13 -0700221 if (e && !get_nat_flag(e, IS_CHECKPOINTED))
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700222 is_cp = false;
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700223 up_read(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900224 return is_cp;
225}
226
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700227bool need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino)
Jaegeuk Kimb6fe5872014-06-04 00:39:42 +0900228{
229 struct f2fs_nm_info *nm_i = NM_I(sbi);
230 struct nat_entry *e;
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700231 bool need_update = true;
Jaegeuk Kimb6fe5872014-06-04 00:39:42 +0900232
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700233 down_read(&nm_i->nat_tree_lock);
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700234 e = __lookup_nat_cache(nm_i, ino);
235 if (e && get_nat_flag(e, HAS_LAST_FSYNC) &&
236 (get_nat_flag(e, IS_CHECKPOINTED) ||
237 get_nat_flag(e, HAS_FSYNCED_INODE)))
238 need_update = false;
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700239 up_read(&nm_i->nat_tree_lock);
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700240 return need_update;
Jaegeuk Kimb6fe5872014-06-04 00:39:42 +0900241}
242
Chao Yu5c9e41842016-12-13 18:54:59 +0800243static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid,
244 bool no_fail)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900245{
246 struct nat_entry *new;
247
Chao Yu5c9e41842016-12-13 18:54:59 +0800248 if (no_fail) {
249 new = f2fs_kmem_cache_alloc(nat_entry_slab, GFP_NOFS);
250 f2fs_radix_tree_insert(&nm_i->nat_root, nid, new);
251 } else {
252 new = kmem_cache_alloc(nat_entry_slab, GFP_NOFS);
253 if (!new)
254 return NULL;
255 if (radix_tree_insert(&nm_i->nat_root, nid, new)) {
256 kmem_cache_free(nat_entry_slab, new);
257 return NULL;
258 }
259 }
260
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900261 memset(new, 0, sizeof(struct nat_entry));
262 nat_set_nid(new, nid);
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700263 nat_reset_flag(new);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900264 list_add_tail(&new->list, &nm_i->nat_entries);
265 nm_i->nat_cnt++;
266 return new;
267}
268
Chao Yu1515aef2016-02-19 18:12:28 +0800269static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid,
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900270 struct f2fs_nat_entry *ne)
271{
Chao Yu1515aef2016-02-19 18:12:28 +0800272 struct f2fs_nm_info *nm_i = NM_I(sbi);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900273 struct nat_entry *e;
Jaegeuk Kim9be32d72014-12-05 10:39:49 -0800274
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900275 e = __lookup_nat_cache(nm_i, nid);
276 if (!e) {
Chao Yu5c9e41842016-12-13 18:54:59 +0800277 e = grab_nat_entry(nm_i, nid, false);
278 if (e)
279 node_info_from_raw_nat(&e->ni, ne);
Chao Yu1515aef2016-02-19 18:12:28 +0800280 } else {
Eric Biggers0c0b4712016-10-11 10:36:12 -0700281 f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) ||
282 nat_get_blkaddr(e) !=
283 le32_to_cpu(ne->block_addr) ||
Chao Yu1515aef2016-02-19 18:12:28 +0800284 nat_get_version(e) != ne->version);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900285 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900286}
287
288static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
Jaegeuk Kim479f40c2014-03-20 21:52:53 +0900289 block_t new_blkaddr, bool fsync_done)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900290{
291 struct f2fs_nm_info *nm_i = NM_I(sbi);
292 struct nat_entry *e;
Jaegeuk Kim9be32d72014-12-05 10:39:49 -0800293
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700294 down_write(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900295 e = __lookup_nat_cache(nm_i, ni->nid);
296 if (!e) {
Chao Yu5c9e41842016-12-13 18:54:59 +0800297 e = grab_nat_entry(nm_i, ni->nid, true);
Chao Yu5c27f4e2014-12-18 17:37:21 +0800298 copy_node_info(&e->ni, ni);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700299 f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900300 } else if (new_blkaddr == NEW_ADDR) {
301 /*
302 * when nid is reallocated,
303 * previous nat entry can be remained in nat cache.
304 * So, reinitialize it with new information.
305 */
Chao Yu5c27f4e2014-12-18 17:37:21 +0800306 copy_node_info(&e->ni, ni);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700307 f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900308 }
309
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900310 /* sanity check */
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700311 f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr);
312 f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR &&
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900313 new_blkaddr == NULL_ADDR);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700314 f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR &&
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900315 new_blkaddr == NEW_ADDR);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700316 f2fs_bug_on(sbi, nat_get_blkaddr(e) != NEW_ADDR &&
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900317 nat_get_blkaddr(e) != NULL_ADDR &&
318 new_blkaddr == NEW_ADDR);
319
arter97e1c42042014-08-06 23:22:50 +0900320 /* increment version no as node is removed */
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900321 if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) {
322 unsigned char version = nat_get_version(e);
323 nat_set_version(e, inc_node_version(version));
Jaegeuk Kim26834462015-08-14 17:57:29 -0700324
325 /* in order to reuse the nid */
326 if (nm_i->next_scan_nid > ni->nid)
327 nm_i->next_scan_nid = ni->nid;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900328 }
329
330 /* change address */
331 nat_set_blkaddr(e, new_blkaddr);
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700332 if (new_blkaddr == NEW_ADDR || new_blkaddr == NULL_ADDR)
333 set_nat_flag(e, IS_CHECKPOINTED, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900334 __set_nat_cache_dirty(nm_i, e);
Jaegeuk Kim479f40c2014-03-20 21:52:53 +0900335
336 /* update fsync_mark if its inode nat entry is still alive */
Chao Yud5b692b2015-04-30 18:35:50 +0800337 if (ni->nid != ni->ino)
338 e = __lookup_nat_cache(nm_i, ni->ino);
Jaegeuk Kim88bd02c2014-09-15 14:50:48 -0700339 if (e) {
340 if (fsync_done && ni->nid == ni->ino)
341 set_nat_flag(e, HAS_FSYNCED_INODE, true);
342 set_nat_flag(e, HAS_LAST_FSYNC, fsync_done);
343 }
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700344 up_write(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900345}
346
Jaegeuk Kim4660f9c2013-10-24 14:19:18 +0900347int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900348{
349 struct f2fs_nm_info *nm_i = NM_I(sbi);
Jaegeuk Kim1b38dc82015-06-19 15:36:07 -0700350 int nr = nr_shrink;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900351
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700352 if (!down_write_trylock(&nm_i->nat_tree_lock))
353 return 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900354
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900355 while (nr_shrink && !list_empty(&nm_i->nat_entries)) {
356 struct nat_entry *ne;
357 ne = list_first_entry(&nm_i->nat_entries,
358 struct nat_entry, list);
359 __del_from_nat_cache(nm_i, ne);
360 nr_shrink--;
361 }
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700362 up_write(&nm_i->nat_tree_lock);
Jaegeuk Kim1b38dc82015-06-19 15:36:07 -0700363 return nr - nr_shrink;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900364}
365
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +0900366/*
arter97e1c42042014-08-06 23:22:50 +0900367 * This function always returns success
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900368 */
369void get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni)
370{
371 struct f2fs_nm_info *nm_i = NM_I(sbi);
372 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
Chao Yub7ad7512016-02-19 18:08:46 +0800373 struct f2fs_journal *journal = curseg->journal;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900374 nid_t start_nid = START_NID(nid);
375 struct f2fs_nat_block *nat_blk;
376 struct page *page = NULL;
377 struct f2fs_nat_entry ne;
378 struct nat_entry *e;
379 int i;
380
381 ni->nid = nid;
382
383 /* Check nat cache */
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700384 down_read(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900385 e = __lookup_nat_cache(nm_i, nid);
386 if (e) {
387 ni->ino = nat_get_ino(e);
388 ni->blk_addr = nat_get_blkaddr(e);
389 ni->version = nat_get_version(e);
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700390 up_read(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900391 return;
Chao Yu1515aef2016-02-19 18:12:28 +0800392 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900393
Jaegeuk Kim3547ea92014-12-30 23:08:26 -0800394 memset(&ne, 0, sizeof(struct f2fs_nat_entry));
395
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900396 /* Check current segment summary */
Chao Yub7ad7512016-02-19 18:08:46 +0800397 down_read(&curseg->journal_rwsem);
Chao Yudfc08a12016-02-14 18:50:40 +0800398 i = lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900399 if (i >= 0) {
Chao Yudfc08a12016-02-14 18:50:40 +0800400 ne = nat_in_journal(journal, i);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900401 node_info_from_raw_nat(ni, &ne);
402 }
Chao Yub7ad7512016-02-19 18:08:46 +0800403 up_read(&curseg->journal_rwsem);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900404 if (i >= 0)
405 goto cache;
406
407 /* Fill node_info from nat page */
408 page = get_current_nat_page(sbi, start_nid);
409 nat_blk = (struct f2fs_nat_block *)page_address(page);
410 ne = nat_blk->entries[nid - start_nid];
411 node_info_from_raw_nat(ni, &ne);
412 f2fs_put_page(page, 1);
413cache:
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700414 up_read(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900415 /* cache nat entry */
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700416 down_write(&nm_i->nat_tree_lock);
Chao Yu1515aef2016-02-19 18:12:28 +0800417 cache_nat_entry(sbi, nid, &ne);
Jaegeuk Kimb873b792016-08-04 11:38:25 -0700418 up_write(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900419}
420
Jaegeuk Kim79344ef2016-05-06 16:19:43 -0700421/*
422 * readahead MAX_RA_NODE number of node pages.
423 */
424static void ra_node_pages(struct page *parent, int start, int n)
425{
426 struct f2fs_sb_info *sbi = F2FS_P_SB(parent);
427 struct blk_plug plug;
428 int i, end;
429 nid_t nid;
430
431 blk_start_plug(&plug);
432
433 /* Then, try readahead for siblings of the desired node */
434 end = start + n;
435 end = min(end, NIDS_PER_BLOCK);
436 for (i = start; i < end; i++) {
437 nid = get_nid(parent, i, false);
438 ra_node_page(sbi, nid);
439 }
440
441 blk_finish_plug(&plug);
442}
443
Chao Yu3cf45742016-01-26 15:40:44 +0800444pgoff_t get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs)
445{
446 const long direct_index = ADDRS_PER_INODE(dn->inode);
447 const long direct_blks = ADDRS_PER_BLOCK;
448 const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK;
449 unsigned int skipped_unit = ADDRS_PER_BLOCK;
450 int cur_level = dn->cur_level;
451 int max_level = dn->max_level;
452 pgoff_t base = 0;
453
454 if (!dn->max_level)
455 return pgofs + 1;
456
457 while (max_level-- > cur_level)
458 skipped_unit *= NIDS_PER_BLOCK;
459
460 switch (dn->max_level) {
461 case 3:
462 base += 2 * indirect_blks;
463 case 2:
464 base += 2 * direct_blks;
465 case 1:
466 base += direct_index;
467 break;
468 default:
469 f2fs_bug_on(F2FS_I_SB(dn->inode), 1);
470 }
471
472 return ((pgofs - base) / skipped_unit + 1) * skipped_unit + base;
473}
474
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +0900475/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900476 * The maximum depth is four.
477 * Offset[0] will have raw inode offset.
478 */
Chao Yu81ca7352016-01-26 15:39:35 +0800479static int get_node_path(struct inode *inode, long block,
Jaegeuk Kimde936532013-08-12 21:08:03 +0900480 int offset[4], unsigned int noffset[4])
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900481{
Chao Yu81ca7352016-01-26 15:39:35 +0800482 const long direct_index = ADDRS_PER_INODE(inode);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900483 const long direct_blks = ADDRS_PER_BLOCK;
484 const long dptrs_per_blk = NIDS_PER_BLOCK;
485 const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK;
486 const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;
487 int n = 0;
488 int level = 0;
489
490 noffset[0] = 0;
491
492 if (block < direct_index) {
Namjae Jeon25c0a6e2013-03-02 12:41:31 +0900493 offset[n] = block;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900494 goto got;
495 }
496 block -= direct_index;
497 if (block < direct_blks) {
498 offset[n++] = NODE_DIR1_BLOCK;
499 noffset[n] = 1;
Namjae Jeon25c0a6e2013-03-02 12:41:31 +0900500 offset[n] = block;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900501 level = 1;
502 goto got;
503 }
504 block -= direct_blks;
505 if (block < direct_blks) {
506 offset[n++] = NODE_DIR2_BLOCK;
507 noffset[n] = 2;
Namjae Jeon25c0a6e2013-03-02 12:41:31 +0900508 offset[n] = block;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900509 level = 1;
510 goto got;
511 }
512 block -= direct_blks;
513 if (block < indirect_blks) {
514 offset[n++] = NODE_IND1_BLOCK;
515 noffset[n] = 3;
516 offset[n++] = block / direct_blks;
517 noffset[n] = 4 + offset[n - 1];
Namjae Jeon25c0a6e2013-03-02 12:41:31 +0900518 offset[n] = block % direct_blks;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900519 level = 2;
520 goto got;
521 }
522 block -= indirect_blks;
523 if (block < indirect_blks) {
524 offset[n++] = NODE_IND2_BLOCK;
525 noffset[n] = 4 + dptrs_per_blk;
526 offset[n++] = block / direct_blks;
527 noffset[n] = 5 + dptrs_per_blk + offset[n - 1];
Namjae Jeon25c0a6e2013-03-02 12:41:31 +0900528 offset[n] = block % direct_blks;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900529 level = 2;
530 goto got;
531 }
532 block -= indirect_blks;
533 if (block < dindirect_blks) {
534 offset[n++] = NODE_DIND_BLOCK;
535 noffset[n] = 5 + (dptrs_per_blk * 2);
536 offset[n++] = block / indirect_blks;
537 noffset[n] = 6 + (dptrs_per_blk * 2) +
538 offset[n - 1] * (dptrs_per_blk + 1);
539 offset[n++] = (block / direct_blks) % dptrs_per_blk;
540 noffset[n] = 7 + (dptrs_per_blk * 2) +
541 offset[n - 2] * (dptrs_per_blk + 1) +
542 offset[n - 1];
Namjae Jeon25c0a6e2013-03-02 12:41:31 +0900543 offset[n] = block % direct_blks;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900544 level = 3;
545 goto got;
546 } else {
547 BUG();
548 }
549got:
550 return level;
551}
552
553/*
554 * Caller should call f2fs_put_dnode(dn).
Chao Yu4f4124d2013-12-21 18:02:14 +0800555 * Also, it should grab and release a rwsem by calling f2fs_lock_op() and
556 * f2fs_unlock_op() only if ro is not set RDONLY_NODE.
Jaegeuk Kim39936832012-11-22 16:21:29 +0900557 * In the case of RDONLY_NODE, we don't need to care about mutex.
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900558 */
Jaegeuk Kim266e97a2013-02-26 13:10:46 +0900559int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900560{
Jaegeuk Kim40813632014-09-02 15:31:18 -0700561 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900562 struct page *npage[4];
Jaegeuk Kimf1a3b982015-02-11 11:25:11 -0800563 struct page *parent = NULL;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900564 int offset[4];
565 unsigned int noffset[4];
566 nid_t nids[4];
Chao Yu3cf45742016-01-26 15:40:44 +0800567 int level, i = 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900568 int err = 0;
569
Chao Yu81ca7352016-01-26 15:39:35 +0800570 level = get_node_path(dn->inode, index, offset, noffset);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900571
572 nids[0] = dn->inode->i_ino;
Jaegeuk Kim1646cfa2013-05-20 09:42:28 +0900573 npage[0] = dn->inode_page;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900574
Jaegeuk Kim1646cfa2013-05-20 09:42:28 +0900575 if (!npage[0]) {
576 npage[0] = get_node_page(sbi, nids[0]);
577 if (IS_ERR(npage[0]))
578 return PTR_ERR(npage[0]);
579 }
Jaegeuk Kimf1a3b982015-02-11 11:25:11 -0800580
581 /* if inline_data is set, should not report any block indices */
582 if (f2fs_has_inline_data(dn->inode) && index) {
Jaegeuk Kim76629162015-03-02 16:28:16 -0800583 err = -ENOENT;
Jaegeuk Kimf1a3b982015-02-11 11:25:11 -0800584 f2fs_put_page(npage[0], 1);
585 goto release_out;
586 }
587
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900588 parent = npage[0];
Changman Lee52c2db32013-02-20 07:47:06 +0900589 if (level != 0)
590 nids[1] = get_nid(parent, offset[0], true);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900591 dn->inode_page = npage[0];
592 dn->inode_page_locked = true;
593
594 /* get indirect or direct nodes */
595 for (i = 1; i <= level; i++) {
596 bool done = false;
597
Jaegeuk Kim266e97a2013-02-26 13:10:46 +0900598 if (!nids[i] && mode == ALLOC_NODE) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900599 /* alloc new node */
600 if (!alloc_nid(sbi, &(nids[i]))) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900601 err = -ENOSPC;
602 goto release_pages;
603 }
604
605 dn->nid = nids[i];
Jaegeuk Kim8ae8f162013-06-03 19:46:19 +0900606 npage[i] = new_node_page(dn, noffset[i], NULL);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900607 if (IS_ERR(npage[i])) {
608 alloc_nid_failed(sbi, nids[i]);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900609 err = PTR_ERR(npage[i]);
610 goto release_pages;
611 }
612
613 set_nid(parent, offset[i - 1], nids[i], i == 1);
614 alloc_nid_done(sbi, nids[i]);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900615 done = true;
Jaegeuk Kim266e97a2013-02-26 13:10:46 +0900616 } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900617 npage[i] = get_node_page_ra(parent, offset[i - 1]);
618 if (IS_ERR(npage[i])) {
619 err = PTR_ERR(npage[i]);
620 goto release_pages;
621 }
622 done = true;
623 }
624 if (i == 1) {
625 dn->inode_page_locked = false;
626 unlock_page(parent);
627 } else {
628 f2fs_put_page(parent, 1);
629 }
630
631 if (!done) {
632 npage[i] = get_node_page(sbi, nids[i]);
633 if (IS_ERR(npage[i])) {
634 err = PTR_ERR(npage[i]);
635 f2fs_put_page(npage[0], 0);
636 goto release_out;
637 }
638 }
639 if (i < level) {
640 parent = npage[i];
641 nids[i + 1] = get_nid(parent, offset[i], false);
642 }
643 }
644 dn->nid = nids[level];
645 dn->ofs_in_node = offset[level];
646 dn->node_page = npage[level];
647 dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
648 return 0;
649
650release_pages:
651 f2fs_put_page(parent, 1);
652 if (i > 1)
653 f2fs_put_page(npage[0], 0);
654release_out:
655 dn->inode_page = NULL;
656 dn->node_page = NULL;
Chao Yu3cf45742016-01-26 15:40:44 +0800657 if (err == -ENOENT) {
658 dn->cur_level = i;
659 dn->max_level = level;
Jaegeuk Kim0a2aa8f2016-07-08 17:42:21 -0700660 dn->ofs_in_node = offset[level];
Chao Yu3cf45742016-01-26 15:40:44 +0800661 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900662 return err;
663}
664
665static void truncate_node(struct dnode_of_data *dn)
666{
Jaegeuk Kim40813632014-09-02 15:31:18 -0700667 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900668 struct node_info ni;
669
670 get_node_info(sbi, dn->nid, &ni);
Jaegeuk Kim71e9fec2012-12-20 15:10:06 +0900671 if (dn->inode->i_blocks == 0) {
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700672 f2fs_bug_on(sbi, ni.blk_addr != NULL_ADDR);
Jaegeuk Kim71e9fec2012-12-20 15:10:06 +0900673 goto invalidate;
674 }
Jaegeuk Kim9850cf42014-09-02 15:52:58 -0700675 f2fs_bug_on(sbi, ni.blk_addr == NULL_ADDR);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900676
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900677 /* Deallocate node address */
Jaegeuk Kim71e9fec2012-12-20 15:10:06 +0900678 invalidate_blocks(sbi, ni.blk_addr);
Gu Zhengef86d702013-11-19 18:03:38 +0800679 dec_valid_node_count(sbi, dn->inode);
Jaegeuk Kim479f40c2014-03-20 21:52:53 +0900680 set_node_addr(sbi, &ni, NULL_ADDR, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900681
682 if (dn->nid == dn->inode->i_ino) {
683 remove_orphan_inode(sbi, dn->nid);
684 dec_valid_inode_count(sbi);
Jaegeuk Kim0f18b462016-05-20 11:10:10 -0700685 f2fs_inode_synced(dn->inode);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900686 }
Jaegeuk Kim71e9fec2012-12-20 15:10:06 +0900687invalidate:
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900688 clear_node_page_dirty(dn->node_page);
Chao Yucaf00472015-01-28 17:48:42 +0800689 set_sbi_flag(sbi, SBI_IS_DIRTY);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900690
691 f2fs_put_page(dn->node_page, 1);
Jaegeuk Kimbf39c002014-01-22 20:41:57 +0900692
693 invalidate_mapping_pages(NODE_MAPPING(sbi),
694 dn->node_page->index, dn->node_page->index);
695
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900696 dn->node_page = NULL;
Namjae Jeon51dd6242013-04-20 01:28:52 +0900697 trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900698}
699
700static int truncate_dnode(struct dnode_of_data *dn)
701{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900702 struct page *page;
703
704 if (dn->nid == 0)
705 return 1;
706
707 /* get direct node */
Jaegeuk Kim40813632014-09-02 15:31:18 -0700708 page = get_node_page(F2FS_I_SB(dn->inode), dn->nid);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900709 if (IS_ERR(page) && PTR_ERR(page) == -ENOENT)
710 return 1;
711 else if (IS_ERR(page))
712 return PTR_ERR(page);
713
714 /* Make dnode_of_data for parameter */
715 dn->node_page = page;
716 dn->ofs_in_node = 0;
717 truncate_data_blocks(dn);
718 truncate_node(dn);
719 return 1;
720}
721
722static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
723 int ofs, int depth)
724{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900725 struct dnode_of_data rdn = *dn;
726 struct page *page;
727 struct f2fs_node *rn;
728 nid_t child_nid;
729 unsigned int child_nofs;
730 int freed = 0;
731 int i, ret;
732
733 if (dn->nid == 0)
734 return NIDS_PER_BLOCK + 1;
735
Namjae Jeon51dd6242013-04-20 01:28:52 +0900736 trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);
737
Jaegeuk Kim40813632014-09-02 15:31:18 -0700738 page = get_node_page(F2FS_I_SB(dn->inode), dn->nid);
Namjae Jeon51dd6242013-04-20 01:28:52 +0900739 if (IS_ERR(page)) {
740 trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(page));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900741 return PTR_ERR(page);
Namjae Jeon51dd6242013-04-20 01:28:52 +0900742 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900743
Jaegeuk Kim79344ef2016-05-06 16:19:43 -0700744 ra_node_pages(page, ofs, NIDS_PER_BLOCK);
745
Gu Zheng45590712013-07-15 17:57:38 +0800746 rn = F2FS_NODE(page);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900747 if (depth < 3) {
748 for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) {
749 child_nid = le32_to_cpu(rn->in.nid[i]);
750 if (child_nid == 0)
751 continue;
752 rdn.nid = child_nid;
753 ret = truncate_dnode(&rdn);
754 if (ret < 0)
755 goto out_err;
Jaegeuk Kim12719ae2016-01-07 13:23:12 -0800756 if (set_nid(page, i, 0, false))
757 dn->node_changed = true;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900758 }
759 } else {
760 child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1;
761 for (i = ofs; i < NIDS_PER_BLOCK; i++) {
762 child_nid = le32_to_cpu(rn->in.nid[i]);
763 if (child_nid == 0) {
764 child_nofs += NIDS_PER_BLOCK + 1;
765 continue;
766 }
767 rdn.nid = child_nid;
768 ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1);
769 if (ret == (NIDS_PER_BLOCK + 1)) {
Jaegeuk Kim12719ae2016-01-07 13:23:12 -0800770 if (set_nid(page, i, 0, false))
771 dn->node_changed = true;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900772 child_nofs += ret;
773 } else if (ret < 0 && ret != -ENOENT) {
774 goto out_err;
775 }
776 }
777 freed = child_nofs;
778 }
779
780 if (!ofs) {
781 /* remove current indirect node */
782 dn->node_page = page;
783 truncate_node(dn);
784 freed++;
785 } else {
786 f2fs_put_page(page, 1);
787 }
Namjae Jeon51dd6242013-04-20 01:28:52 +0900788 trace_f2fs_truncate_nodes_exit(dn->inode, freed);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900789 return freed;
790
791out_err:
792 f2fs_put_page(page, 1);
Namjae Jeon51dd6242013-04-20 01:28:52 +0900793 trace_f2fs_truncate_nodes_exit(dn->inode, ret);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900794 return ret;
795}
796
797static int truncate_partial_nodes(struct dnode_of_data *dn,
798 struct f2fs_inode *ri, int *offset, int depth)
799{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900800 struct page *pages[2];
801 nid_t nid[3];
802 nid_t child_nid;
803 int err = 0;
804 int i;
805 int idx = depth - 2;
806
807 nid[0] = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
808 if (!nid[0])
809 return 0;
810
811 /* get indirect nodes in the path */
shifei10.gea225dca2013-10-29 15:32:34 +0800812 for (i = 0; i < idx + 1; i++) {
arter97e1c42042014-08-06 23:22:50 +0900813 /* reference count'll be increased */
Jaegeuk Kim40813632014-09-02 15:31:18 -0700814 pages[i] = get_node_page(F2FS_I_SB(dn->inode), nid[i]);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900815 if (IS_ERR(pages[i])) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900816 err = PTR_ERR(pages[i]);
shifei10.gea225dca2013-10-29 15:32:34 +0800817 idx = i - 1;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900818 goto fail;
819 }
820 nid[i + 1] = get_nid(pages[i], offset[i + 1], false);
821 }
822
Jaegeuk Kim79344ef2016-05-06 16:19:43 -0700823 ra_node_pages(pages[idx], offset[idx + 1], NIDS_PER_BLOCK);
824
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900825 /* free direct nodes linked to a partial indirect node */
shifei10.gea225dca2013-10-29 15:32:34 +0800826 for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900827 child_nid = get_nid(pages[idx], i, false);
828 if (!child_nid)
829 continue;
830 dn->nid = child_nid;
831 err = truncate_dnode(dn);
832 if (err < 0)
833 goto fail;
Jaegeuk Kim12719ae2016-01-07 13:23:12 -0800834 if (set_nid(pages[idx], i, 0, false))
835 dn->node_changed = true;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900836 }
837
shifei10.gea225dca2013-10-29 15:32:34 +0800838 if (offset[idx + 1] == 0) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900839 dn->node_page = pages[idx];
840 dn->nid = nid[idx];
841 truncate_node(dn);
842 } else {
843 f2fs_put_page(pages[idx], 1);
844 }
845 offset[idx]++;
shifei10.gea225dca2013-10-29 15:32:34 +0800846 offset[idx + 1] = 0;
847 idx--;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900848fail:
shifei10.gea225dca2013-10-29 15:32:34 +0800849 for (i = idx; i >= 0; i--)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900850 f2fs_put_page(pages[i], 1);
Namjae Jeon51dd6242013-04-20 01:28:52 +0900851
852 trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);
853
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900854 return err;
855}
856
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +0900857/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900858 * All the block addresses of data and nodes should be nullified.
859 */
860int truncate_inode_blocks(struct inode *inode, pgoff_t from)
861{
Jaegeuk Kim40813632014-09-02 15:31:18 -0700862 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900863 int err = 0, cont = 1;
864 int level, offset[4], noffset[4];
Jaegeuk Kim7dd690c2013-02-12 07:28:55 +0900865 unsigned int nofs = 0;
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +0900866 struct f2fs_inode *ri;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900867 struct dnode_of_data dn;
868 struct page *page;
869
Namjae Jeon51dd6242013-04-20 01:28:52 +0900870 trace_f2fs_truncate_inode_blocks_enter(inode, from);
871
Chao Yu81ca7352016-01-26 15:39:35 +0800872 level = get_node_path(inode, from, offset, noffset);
Jaegeuk Kimff373552016-03-29 16:13:45 -0700873
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900874 page = get_node_page(sbi, inode->i_ino);
Namjae Jeon51dd6242013-04-20 01:28:52 +0900875 if (IS_ERR(page)) {
876 trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(page));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900877 return PTR_ERR(page);
Namjae Jeon51dd6242013-04-20 01:28:52 +0900878 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900879
880 set_new_dnode(&dn, inode, page, NULL, 0);
881 unlock_page(page);
882
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +0900883 ri = F2FS_INODE(page);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900884 switch (level) {
885 case 0:
886 case 1:
887 nofs = noffset[1];
888 break;
889 case 2:
890 nofs = noffset[1];
891 if (!offset[level - 1])
892 goto skip_partial;
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +0900893 err = truncate_partial_nodes(&dn, ri, offset, level);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900894 if (err < 0 && err != -ENOENT)
895 goto fail;
896 nofs += 1 + NIDS_PER_BLOCK;
897 break;
898 case 3:
899 nofs = 5 + 2 * NIDS_PER_BLOCK;
900 if (!offset[level - 1])
901 goto skip_partial;
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +0900902 err = truncate_partial_nodes(&dn, ri, offset, level);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900903 if (err < 0 && err != -ENOENT)
904 goto fail;
905 break;
906 default:
907 BUG();
908 }
909
910skip_partial:
911 while (cont) {
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +0900912 dn.nid = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900913 switch (offset[0]) {
914 case NODE_DIR1_BLOCK:
915 case NODE_DIR2_BLOCK:
916 err = truncate_dnode(&dn);
917 break;
918
919 case NODE_IND1_BLOCK:
920 case NODE_IND2_BLOCK:
921 err = truncate_nodes(&dn, nofs, offset[1], 2);
922 break;
923
924 case NODE_DIND_BLOCK:
925 err = truncate_nodes(&dn, nofs, offset[1], 3);
926 cont = 0;
927 break;
928
929 default:
930 BUG();
931 }
932 if (err < 0 && err != -ENOENT)
933 goto fail;
934 if (offset[1] == 0 &&
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +0900935 ri->i_nid[offset[0] - NODE_DIR1_BLOCK]) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900936 lock_page(page);
Jaegeuk Kimff373552016-03-29 16:13:45 -0700937 BUG_ON(page->mapping != NODE_MAPPING(sbi));
Jaegeuk Kimfec1d652016-01-20 23:43:51 +0800938 f2fs_wait_on_page_writeback(page, NODE, true);
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +0900939 ri->i_nid[offset[0] - NODE_DIR1_BLOCK] = 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900940 set_page_dirty(page);
941 unlock_page(page);
942 }
943 offset[1] = 0;
944 offset[0]++;
945 nofs += err;
946 }
947fail:
948 f2fs_put_page(page, 0);
Namjae Jeon51dd6242013-04-20 01:28:52 +0900949 trace_f2fs_truncate_inode_blocks_exit(inode, err);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900950 return err > 0 ? 0 : err;
951}
952
Jaegeuk Kim4f16fb02013-08-14 20:40:06 +0900953int truncate_xattr_node(struct inode *inode, struct page *page)
954{
Jaegeuk Kim40813632014-09-02 15:31:18 -0700955 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kim4f16fb02013-08-14 20:40:06 +0900956 nid_t nid = F2FS_I(inode)->i_xattr_nid;
957 struct dnode_of_data dn;
958 struct page *npage;
959
960 if (!nid)
961 return 0;
962
963 npage = get_node_page(sbi, nid);
964 if (IS_ERR(npage))
965 return PTR_ERR(npage);
966
Jaegeuk Kim205b9822016-05-20 09:52:20 -0700967 f2fs_i_xnid_write(inode, 0);
Jaegeuk Kim65985d92013-08-14 21:57:27 +0900968
Jaegeuk Kim4f16fb02013-08-14 20:40:06 +0900969 set_new_dnode(&dn, inode, page, npage, nid);
970
971 if (page)
Chao Yu01d2d1a2013-11-28 15:43:07 +0800972 dn.inode_page_locked = true;
Jaegeuk Kim4f16fb02013-08-14 20:40:06 +0900973 truncate_node(&dn);
974 return 0;
975}
976
Jaegeuk Kim39936832012-11-22 16:21:29 +0900977/*
Chao Yu4f4124d2013-12-21 18:02:14 +0800978 * Caller should grab and release a rwsem by calling f2fs_lock_op() and
979 * f2fs_unlock_op().
Jaegeuk Kim39936832012-11-22 16:21:29 +0900980 */
Chao Yu13ec7292015-08-24 17:40:45 +0800981int remove_inode_page(struct inode *inode)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900982{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900983 struct dnode_of_data dn;
Chao Yu13ec7292015-08-24 17:40:45 +0800984 int err;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900985
Jaegeuk Kimc2e69582014-08-25 14:45:59 -0700986 set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
Chao Yu13ec7292015-08-24 17:40:45 +0800987 err = get_dnode_of_data(&dn, 0, LOOKUP_NODE);
988 if (err)
989 return err;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900990
Chao Yu13ec7292015-08-24 17:40:45 +0800991 err = truncate_xattr_node(inode, dn.inode_page);
992 if (err) {
Jaegeuk Kimc2e69582014-08-25 14:45:59 -0700993 f2fs_put_dnode(&dn);
Chao Yu13ec7292015-08-24 17:40:45 +0800994 return err;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +0900995 }
Jaegeuk Kimc2e69582014-08-25 14:45:59 -0700996
997 /* remove potential inline_data blocks */
998 if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
999 S_ISLNK(inode->i_mode))
1000 truncate_data_blocks_range(&dn, 1);
1001
arter97e1c42042014-08-06 23:22:50 +09001002 /* 0 is possible, after f2fs_new_inode() has failed */
Jaegeuk Kim9850cf42014-09-02 15:52:58 -07001003 f2fs_bug_on(F2FS_I_SB(inode),
1004 inode->i_blocks != 0 && inode->i_blocks != 1);
Jaegeuk Kimc2e69582014-08-25 14:45:59 -07001005
1006 /* will put inode & node pages */
Jaegeuk Kim71e9fec2012-12-20 15:10:06 +09001007 truncate_node(&dn);
Chao Yu13ec7292015-08-24 17:40:45 +08001008 return 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001009}
1010
Jaegeuk Kima014e032014-06-20 21:44:02 -07001011struct page *new_inode_page(struct inode *inode)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001012{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001013 struct dnode_of_data dn;
1014
1015 /* allocate inode page for new inode */
1016 set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
Jaegeuk Kim44a83ff2013-05-20 10:10:29 +09001017
1018 /* caller should f2fs_put_page(page, 1); */
Jaegeuk Kim8ae8f162013-06-03 19:46:19 +09001019 return new_node_page(&dn, 0, NULL);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001020}
1021
Jaegeuk Kim8ae8f162013-06-03 19:46:19 +09001022struct page *new_node_page(struct dnode_of_data *dn,
1023 unsigned int ofs, struct page *ipage)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001024{
Jaegeuk Kim40813632014-09-02 15:31:18 -07001025 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
Jaegeuk Kim25cc5d32017-02-13 17:02:44 -08001026 struct node_info new_ni;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001027 struct page *page;
1028 int err;
1029
Jaegeuk Kim91942322016-05-20 10:13:22 -07001030 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001031 return ERR_PTR(-EPERM);
1032
Jaegeuk Kim300e1292016-04-29 16:11:53 -07001033 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), dn->nid, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001034 if (!page)
1035 return ERR_PTR(-ENOMEM);
1036
Jaegeuk Kim6bacf522013-12-06 15:00:58 +09001037 if (unlikely(!inc_valid_node_count(sbi, dn->inode))) {
Jaegeuk Kim9c027402013-08-12 16:00:46 +09001038 err = -ENOSPC;
1039 goto fail;
1040 }
Jaegeuk Kim25cc5d32017-02-13 17:02:44 -08001041#ifdef CONFIG_F2FS_CHECK_FS
1042 get_node_info(sbi, dn->nid, &new_ni);
1043 f2fs_bug_on(sbi, new_ni.blk_addr != NULL_ADDR);
1044#endif
1045 new_ni.nid = dn->nid;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001046 new_ni.ino = dn->inode->i_ino;
Jaegeuk Kim25cc5d32017-02-13 17:02:44 -08001047 new_ni.blk_addr = NULL_ADDR;
1048 new_ni.flag = 0;
1049 new_ni.version = 0;
Jaegeuk Kim479f40c2014-03-20 21:52:53 +09001050 set_node_addr(sbi, &new_ni, NEW_ADDR, false);
Jaegeuk Kim9c027402013-08-12 16:00:46 +09001051
Jaegeuk Kimfec1d652016-01-20 23:43:51 +08001052 f2fs_wait_on_page_writeback(page, NODE, true);
Jaegeuk Kim9c027402013-08-12 16:00:46 +09001053 fill_node_footer(page, dn->nid, dn->inode->i_ino, ofs, true);
Jaegeuk Kim398b1ac2012-12-19 15:28:39 +09001054 set_cold_node(dn->inode, page);
Jaegeuk Kim237c0792016-06-30 18:49:15 -07001055 if (!PageUptodate(page))
1056 SetPageUptodate(page);
Jaegeuk Kim12719ae2016-01-07 13:23:12 -08001057 if (set_page_dirty(page))
1058 dn->node_changed = true;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001059
Chao Yu4bc8e9b2014-03-17 16:35:06 +08001060 if (f2fs_has_xattr_block(ofs))
Jaegeuk Kim205b9822016-05-20 09:52:20 -07001061 f2fs_i_xnid_write(dn->inode, dn->nid);
Jaegeuk Kim479bd732013-08-12 16:04:53 +09001062
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001063 if (ofs == 0)
1064 inc_valid_inode_count(sbi);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001065 return page;
1066
1067fail:
Jaegeuk Kim71e9fec2012-12-20 15:10:06 +09001068 clear_node_page_dirty(page);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001069 f2fs_put_page(page, 1);
1070 return ERR_PTR(err);
1071}
1072
Jaegeuk Kim56ae6742013-03-31 12:47:20 +09001073/*
1074 * Caller should do after getting the following values.
1075 * 0: f2fs_put_page(page, 0)
Jaegeuk Kim86531d62015-07-15 13:08:21 -07001076 * LOCKED_PAGE or error: f2fs_put_page(page, 1)
Jaegeuk Kim56ae6742013-03-31 12:47:20 +09001077 */
Mike Christie04d328d2016-06-05 14:31:55 -05001078static int read_node_page(struct page *page, int op_flags)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001079{
Jaegeuk Kim40813632014-09-02 15:31:18 -07001080 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001081 struct node_info ni;
Jaegeuk Kimcf04e8e2014-12-17 19:33:13 -08001082 struct f2fs_io_info fio = {
Jaegeuk Kim05ca3632015-04-23 14:38:15 -07001083 .sbi = sbi,
Jaegeuk Kimcf04e8e2014-12-17 19:33:13 -08001084 .type = NODE,
Mike Christie04d328d2016-06-05 14:31:55 -05001085 .op = REQ_OP_READ,
1086 .op_flags = op_flags,
Jaegeuk Kim05ca3632015-04-23 14:38:15 -07001087 .page = page,
Jaegeuk Kim4375a332015-04-23 12:04:33 -07001088 .encrypted_page = NULL,
Jaegeuk Kimcf04e8e2014-12-17 19:33:13 -08001089 };
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001090
Jaegeuk Kim3bdad3c2016-06-30 19:04:16 -07001091 if (PageUptodate(page))
1092 return LOCKED_PAGE;
1093
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001094 get_node_info(sbi, page->index, &ni);
1095
Jaegeuk Kim6bacf522013-12-06 15:00:58 +09001096 if (unlikely(ni.blk_addr == NULL_ADDR)) {
Jaegeuk Kim2bca1e22015-02-25 19:25:01 -08001097 ClearPageUptodate(page);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001098 return -ENOENT;
Jaegeuk Kim393ff912013-03-08 21:29:23 +09001099 }
1100
Chao Yu7a9d7542016-02-22 18:36:38 +08001101 fio.new_blkaddr = fio.old_blkaddr = ni.blk_addr;
Jaegeuk Kim05ca3632015-04-23 14:38:15 -07001102 return f2fs_submit_page_bio(&fio);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001103}
1104
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +09001105/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001106 * Readahead a node page
1107 */
1108void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
1109{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001110 struct page *apage;
Jaegeuk Kim56ae6742013-03-31 12:47:20 +09001111 int err;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001112
Chao Yue8458722016-01-08 20:13:37 +08001113 if (!nid)
1114 return;
1115 f2fs_bug_on(sbi, check_nid_range(sbi, nid));
1116
Fan Li999270d2016-02-29 14:29:51 +08001117 rcu_read_lock();
1118 apage = radix_tree_lookup(&NODE_MAPPING(sbi)->page_tree, nid);
1119 rcu_read_unlock();
1120 if (apage)
Jaegeuk Kim393ff912013-03-08 21:29:23 +09001121 return;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001122
Jaegeuk Kim300e1292016-04-29 16:11:53 -07001123 apage = f2fs_grab_cache_page(NODE_MAPPING(sbi), nid, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001124 if (!apage)
1125 return;
1126
Christoph Hellwig70246282016-07-19 11:28:41 +02001127 err = read_node_page(apage, REQ_RAHEAD);
Jaegeuk Kim86531d62015-07-15 13:08:21 -07001128 f2fs_put_page(apage, err ? 1 : 0);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001129}
1130
Jaegeuk Kim17a0ee52016-03-08 09:04:35 -08001131static struct page *__get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid,
Chao Yu0e022ea2016-01-05 16:52:46 +08001132 struct page *parent, int start)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001133{
Jaegeuk Kim56ae6742013-03-31 12:47:20 +09001134 struct page *page;
1135 int err;
Jaegeuk Kim4aa69d52015-12-23 14:17:47 -08001136
1137 if (!nid)
1138 return ERR_PTR(-ENOENT);
1139 f2fs_bug_on(sbi, check_nid_range(sbi, nid));
Jaegeuk Kimafcb7ca02013-04-26 11:55:17 +09001140repeat:
Jaegeuk Kim300e1292016-04-29 16:11:53 -07001141 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), nid, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001142 if (!page)
1143 return ERR_PTR(-ENOMEM);
1144
Christoph Hellwig70fd7612016-11-01 07:40:10 -06001145 err = read_node_page(page, 0);
Jaegeuk Kim86531d62015-07-15 13:08:21 -07001146 if (err < 0) {
1147 f2fs_put_page(page, 1);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001148 return ERR_PTR(err);
Chao Yue1c51b9f2015-12-11 16:08:22 +08001149 } else if (err == LOCKED_PAGE) {
1150 goto page_hit;
Jaegeuk Kim86531d62015-07-15 13:08:21 -07001151 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001152
Chao Yu0e022ea2016-01-05 16:52:46 +08001153 if (parent)
Jaegeuk Kim79344ef2016-05-06 16:19:43 -07001154 ra_node_pages(parent, start + 1, MAX_RA_NODE);
Chao Yu0e022ea2016-01-05 16:52:46 +08001155
Chao Yue1c51b9f2015-12-11 16:08:22 +08001156 lock_page(page);
1157
Jaegeuk Kim4ef51a82014-01-21 18:51:16 +09001158 if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
Jaegeuk Kimafcb7ca02013-04-26 11:55:17 +09001159 f2fs_put_page(page, 1);
1160 goto repeat;
1161 }
Chao Yu1563ac72016-07-03 22:05:12 +08001162
1163 if (unlikely(!PageUptodate(page)))
1164 goto out_err;
Chao Yue1c51b9f2015-12-11 16:08:22 +08001165page_hit:
Yunlong Song0c9df7f2016-05-26 19:40:29 +08001166 if(unlikely(nid != nid_of_node(page))) {
1167 f2fs_bug_on(sbi, 1);
1168 ClearPageUptodate(page);
1169out_err:
1170 f2fs_put_page(page, 1);
1171 return ERR_PTR(-EIO);
1172 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001173 return page;
1174}
1175
Chao Yu0e022ea2016-01-05 16:52:46 +08001176struct page *get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid)
1177{
1178 return __get_node_page(sbi, nid, NULL, 0);
1179}
1180
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001181struct page *get_node_page_ra(struct page *parent, int start)
1182{
Jaegeuk Kim40813632014-09-02 15:31:18 -07001183 struct f2fs_sb_info *sbi = F2FS_P_SB(parent);
Chao Yu0e022ea2016-01-05 16:52:46 +08001184 nid_t nid = get_nid(parent, start, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001185
Chao Yu0e022ea2016-01-05 16:52:46 +08001186 return __get_node_page(sbi, nid, parent, start);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001187}
1188
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001189static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino)
1190{
1191 struct inode *inode;
1192 struct page *page;
Chao Yu0f3311a2016-05-21 00:11:09 +08001193 int ret;
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001194
1195 /* should flush inline_data before evict_inode */
1196 inode = ilookup(sbi->sb, ino);
1197 if (!inode)
1198 return;
1199
Jaegeuk Kim4a6de502016-03-30 11:25:31 -07001200 page = pagecache_get_page(inode->i_mapping, 0, FGP_LOCK|FGP_NOWAIT, 0);
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001201 if (!page)
1202 goto iput_out;
1203
1204 if (!PageUptodate(page))
1205 goto page_out;
1206
1207 if (!PageDirty(page))
1208 goto page_out;
1209
1210 if (!clear_page_dirty_for_io(page))
1211 goto page_out;
1212
Chao Yu0f3311a2016-05-21 00:11:09 +08001213 ret = f2fs_write_inline_data(inode, page);
1214 inode_dec_dirty_pages(inode);
Chao Yu933439c2016-10-11 22:57:01 +08001215 remove_dirty_inode(inode);
Chao Yu0f3311a2016-05-21 00:11:09 +08001216 if (ret)
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001217 set_page_dirty(page);
1218page_out:
Jaegeuk Kim4a6de502016-03-30 11:25:31 -07001219 f2fs_put_page(page, 1);
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001220iput_out:
1221 iput(inode);
1222}
1223
Chao Yuda011cc2016-04-27 21:40:15 +08001224void move_node_page(struct page *node_page, int gc_type)
1225{
1226 if (gc_type == FG_GC) {
1227 struct f2fs_sb_info *sbi = F2FS_P_SB(node_page);
1228 struct writeback_control wbc = {
1229 .sync_mode = WB_SYNC_ALL,
1230 .nr_to_write = 1,
1231 .for_reclaim = 0,
1232 };
1233
1234 set_page_dirty(node_page);
1235 f2fs_wait_on_page_writeback(node_page, NODE, true);
1236
1237 f2fs_bug_on(sbi, PageWriteback(node_page));
1238 if (!clear_page_dirty_for_io(node_page))
1239 goto out_page;
1240
1241 if (NODE_MAPPING(sbi)->a_ops->writepage(node_page, &wbc))
1242 unlock_page(node_page);
1243 goto release_page;
1244 } else {
1245 /* set page dirty and write it */
1246 if (!PageWriteback(node_page))
1247 set_page_dirty(node_page);
1248 }
1249out_page:
1250 unlock_page(node_page);
1251release_page:
1252 f2fs_put_page(node_page, 0);
1253}
1254
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001255static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001256{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001257 pgoff_t index, end;
1258 struct pagevec pvec;
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001259 struct page *last_page = NULL;
Jaegeuk Kim52681372016-04-13 16:24:44 -07001260
1261 pagevec_init(&pvec, 0);
1262 index = 0;
1263 end = ULONG_MAX;
1264
1265 while (index <= end) {
1266 int i, nr_pages;
1267 nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
1268 PAGECACHE_TAG_DIRTY,
1269 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1270 if (nr_pages == 0)
1271 break;
1272
1273 for (i = 0; i < nr_pages; i++) {
1274 struct page *page = pvec.pages[i];
1275
1276 if (unlikely(f2fs_cp_error(sbi))) {
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001277 f2fs_put_page(last_page, 0);
Jaegeuk Kim52681372016-04-13 16:24:44 -07001278 pagevec_release(&pvec);
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001279 return ERR_PTR(-EIO);
Jaegeuk Kim52681372016-04-13 16:24:44 -07001280 }
1281
1282 if (!IS_DNODE(page) || !is_cold_node(page))
1283 continue;
1284 if (ino_of_node(page) != ino)
1285 continue;
1286
1287 lock_page(page);
1288
1289 if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
1290continue_unlock:
1291 unlock_page(page);
1292 continue;
1293 }
1294 if (ino_of_node(page) != ino)
1295 goto continue_unlock;
1296
1297 if (!PageDirty(page)) {
1298 /* someone wrote it for us */
1299 goto continue_unlock;
1300 }
1301
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001302 if (last_page)
1303 f2fs_put_page(last_page, 0);
1304
1305 get_page(page);
1306 last_page = page;
1307 unlock_page(page);
1308 }
1309 pagevec_release(&pvec);
1310 cond_resched();
1311 }
1312 return last_page;
1313}
1314
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001315static int __write_node_page(struct page *page, bool atomic, bool *submitted,
Jaegeuk Kimfaa24892017-02-02 18:27:17 -08001316 struct writeback_control *wbc)
1317{
1318 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1319 nid_t nid;
1320 struct node_info ni;
1321 struct f2fs_io_info fio = {
1322 .sbi = sbi,
1323 .type = NODE,
1324 .op = REQ_OP_WRITE,
1325 .op_flags = wbc_to_write_flags(wbc),
1326 .page = page,
1327 .encrypted_page = NULL,
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001328 .submitted = false,
Jaegeuk Kimfaa24892017-02-02 18:27:17 -08001329 };
1330
1331 trace_f2fs_writepage(page, NODE);
1332
1333 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1334 goto redirty_out;
1335 if (unlikely(f2fs_cp_error(sbi)))
1336 goto redirty_out;
1337
1338 /* get old block addr of this node page */
1339 nid = nid_of_node(page);
1340 f2fs_bug_on(sbi, page->index != nid);
1341
1342 if (wbc->for_reclaim) {
1343 if (!down_read_trylock(&sbi->node_write))
1344 goto redirty_out;
1345 } else {
1346 down_read(&sbi->node_write);
1347 }
1348
1349 get_node_info(sbi, nid, &ni);
1350
1351 /* This page is already truncated */
1352 if (unlikely(ni.blk_addr == NULL_ADDR)) {
1353 ClearPageUptodate(page);
1354 dec_page_count(sbi, F2FS_DIRTY_NODES);
1355 up_read(&sbi->node_write);
1356 unlock_page(page);
1357 return 0;
1358 }
1359
Jaegeuk Kime7c75ab2017-02-02 18:18:06 -08001360 if (atomic && !test_opt(sbi, NOBARRIER))
1361 fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
1362
Jaegeuk Kimfaa24892017-02-02 18:27:17 -08001363 set_page_writeback(page);
1364 fio.old_blkaddr = ni.blk_addr;
1365 write_node_page(nid, &fio);
1366 set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(page));
1367 dec_page_count(sbi, F2FS_DIRTY_NODES);
1368 up_read(&sbi->node_write);
1369
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001370 if (wbc->for_reclaim) {
Jaegeuk Kim942fd312017-02-01 16:51:22 -08001371 f2fs_submit_merged_bio_cond(sbi, page->mapping->host, 0,
1372 page->index, NODE, WRITE);
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001373 submitted = NULL;
1374 }
Jaegeuk Kimfaa24892017-02-02 18:27:17 -08001375
1376 unlock_page(page);
1377
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001378 if (unlikely(f2fs_cp_error(sbi))) {
Jaegeuk Kimfaa24892017-02-02 18:27:17 -08001379 f2fs_submit_merged_bio(sbi, NODE, WRITE);
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001380 submitted = NULL;
1381 }
1382 if (submitted)
1383 *submitted = fio.submitted;
Jaegeuk Kimfaa24892017-02-02 18:27:17 -08001384
1385 return 0;
1386
1387redirty_out:
1388 redirty_page_for_writepage(wbc, page);
1389 return AOP_WRITEPAGE_ACTIVATE;
1390}
1391
1392static int f2fs_write_node_page(struct page *page,
1393 struct writeback_control *wbc)
1394{
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001395 return __write_node_page(page, false, NULL, wbc);
Jaegeuk Kimfaa24892017-02-02 18:27:17 -08001396}
1397
Jaegeuk Kim26de9b12016-05-20 20:42:37 -07001398int fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001399 struct writeback_control *wbc, bool atomic)
1400{
1401 pgoff_t index, end;
Jaegeuk Kim942fd312017-02-01 16:51:22 -08001402 pgoff_t last_idx = ULONG_MAX;
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001403 struct pagevec pvec;
1404 int ret = 0;
1405 struct page *last_page = NULL;
1406 bool marked = false;
Jaegeuk Kim26de9b12016-05-20 20:42:37 -07001407 nid_t ino = inode->i_ino;
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001408
1409 if (atomic) {
1410 last_page = last_fsync_dnode(sbi, ino);
1411 if (IS_ERR_OR_NULL(last_page))
1412 return PTR_ERR_OR_ZERO(last_page);
1413 }
1414retry:
1415 pagevec_init(&pvec, 0);
1416 index = 0;
1417 end = ULONG_MAX;
1418
1419 while (index <= end) {
1420 int i, nr_pages;
1421 nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
1422 PAGECACHE_TAG_DIRTY,
1423 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1424 if (nr_pages == 0)
1425 break;
1426
1427 for (i = 0; i < nr_pages; i++) {
1428 struct page *page = pvec.pages[i];
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001429 bool submitted = false;
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001430
1431 if (unlikely(f2fs_cp_error(sbi))) {
1432 f2fs_put_page(last_page, 0);
1433 pagevec_release(&pvec);
Chao Yu9de69272016-10-11 22:57:06 +08001434 ret = -EIO;
1435 goto out;
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001436 }
1437
1438 if (!IS_DNODE(page) || !is_cold_node(page))
1439 continue;
1440 if (ino_of_node(page) != ino)
1441 continue;
1442
1443 lock_page(page);
1444
1445 if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
1446continue_unlock:
1447 unlock_page(page);
1448 continue;
1449 }
1450 if (ino_of_node(page) != ino)
Jaegeuk Kim52681372016-04-13 16:24:44 -07001451 goto continue_unlock;
1452
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001453 if (!PageDirty(page) && page != last_page) {
1454 /* someone wrote it for us */
1455 goto continue_unlock;
1456 }
1457
1458 f2fs_wait_on_page_writeback(page, NODE, true);
1459 BUG_ON(PageWriteback(page));
1460
1461 if (!atomic || page == last_page) {
1462 set_fsync_mark(page, 1);
Jaegeuk Kim26de9b12016-05-20 20:42:37 -07001463 if (IS_INODE(page)) {
1464 if (is_inode_flag_set(inode,
1465 FI_DIRTY_INODE))
1466 update_inode(inode, page);
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001467 set_dentry_mark(page,
Jaegeuk Kim52681372016-04-13 16:24:44 -07001468 need_dentry_mark(sbi, ino));
Jaegeuk Kim26de9b12016-05-20 20:42:37 -07001469 }
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001470 /* may be written by other thread */
1471 if (!PageDirty(page))
1472 set_page_dirty(page);
1473 }
1474
1475 if (!clear_page_dirty_for_io(page))
1476 goto continue_unlock;
Jaegeuk Kim52681372016-04-13 16:24:44 -07001477
Jaegeuk Kime7c75ab2017-02-02 18:18:06 -08001478 ret = __write_node_page(page, atomic &&
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001479 page == last_page,
1480 &submitted, wbc);
Jaegeuk Kimc267ec12016-04-15 09:25:04 -07001481 if (ret) {
Jaegeuk Kim52681372016-04-13 16:24:44 -07001482 unlock_page(page);
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001483 f2fs_put_page(last_page, 0);
1484 break;
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001485 } else if (submitted) {
Jaegeuk Kim942fd312017-02-01 16:51:22 -08001486 last_idx = page->index;
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001487 }
Chao Yu3f5f4952016-09-29 18:50:10 +08001488
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001489 if (page == last_page) {
1490 f2fs_put_page(page, 0);
1491 marked = true;
Jaegeuk Kim52681372016-04-13 16:24:44 -07001492 break;
Jaegeuk Kimc267ec12016-04-15 09:25:04 -07001493 }
Jaegeuk Kim52681372016-04-13 16:24:44 -07001494 }
1495 pagevec_release(&pvec);
1496 cond_resched();
1497
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001498 if (ret || marked)
Jaegeuk Kim52681372016-04-13 16:24:44 -07001499 break;
1500 }
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001501 if (!ret && atomic && !marked) {
1502 f2fs_msg(sbi->sb, KERN_DEBUG,
1503 "Retry to write fsync mark: ino=%u, idx=%lx",
1504 ino, last_page->index);
1505 lock_page(last_page);
Yunlei Hed40a43a2016-11-16 17:26:24 +08001506 f2fs_wait_on_page_writeback(last_page, NODE, true);
Jaegeuk Kim608514d2016-04-15 09:43:17 -07001507 set_page_dirty(last_page);
1508 unlock_page(last_page);
1509 goto retry;
1510 }
Chao Yu9de69272016-10-11 22:57:06 +08001511out:
Jaegeuk Kim942fd312017-02-01 16:51:22 -08001512 if (last_idx != ULONG_MAX)
1513 f2fs_submit_merged_bio_cond(sbi, NULL, ino, last_idx,
1514 NODE, WRITE);
Jaegeuk Kimc267ec12016-04-15 09:25:04 -07001515 return ret ? -EIO: 0;
Jaegeuk Kim52681372016-04-13 16:24:44 -07001516}
1517
1518int sync_node_pages(struct f2fs_sb_info *sbi, struct writeback_control *wbc)
1519{
1520 pgoff_t index, end;
1521 struct pagevec pvec;
1522 int step = 0;
Jaegeuk Kim12bb0a82016-03-11 15:33:22 -08001523 int nwritten = 0;
Chao Yu3f5f4952016-09-29 18:50:10 +08001524 int ret = 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001525
1526 pagevec_init(&pvec, 0);
1527
1528next_step:
1529 index = 0;
Chao Yu80dd9c02016-02-24 17:20:44 +08001530 end = ULONG_MAX;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001531
1532 while (index <= end) {
1533 int i, nr_pages;
Jaegeuk Kim4ef51a82014-01-21 18:51:16 +09001534 nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001535 PAGECACHE_TAG_DIRTY,
1536 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1537 if (nr_pages == 0)
1538 break;
1539
1540 for (i = 0; i < nr_pages; i++) {
1541 struct page *page = pvec.pages[i];
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001542 bool submitted = false;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001543
Chao Yu6d5a1492015-12-24 18:04:56 +08001544 if (unlikely(f2fs_cp_error(sbi))) {
1545 pagevec_release(&pvec);
Chao Yu3f5f4952016-09-29 18:50:10 +08001546 ret = -EIO;
1547 goto out;
Chao Yu6d5a1492015-12-24 18:04:56 +08001548 }
1549
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001550 /*
1551 * flushing sequence with step:
1552 * 0. indirect nodes
1553 * 1. dentry dnodes
1554 * 2. file dnodes
1555 */
1556 if (step == 0 && IS_DNODE(page))
1557 continue;
1558 if (step == 1 && (!IS_DNODE(page) ||
1559 is_cold_node(page)))
1560 continue;
1561 if (step == 2 && (!IS_DNODE(page) ||
1562 !is_cold_node(page)))
1563 continue;
Chao Yu9a4cbc92016-02-22 18:35:46 +08001564lock_node:
Jaegeuk Kim52681372016-04-13 16:24:44 -07001565 if (!trylock_page(page))
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001566 continue;
1567
Jaegeuk Kim4ef51a82014-01-21 18:51:16 +09001568 if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001569continue_unlock:
1570 unlock_page(page);
1571 continue;
1572 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001573
1574 if (!PageDirty(page)) {
1575 /* someone wrote it for us */
1576 goto continue_unlock;
1577 }
1578
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001579 /* flush inline_data */
Jaegeuk Kim52681372016-04-13 16:24:44 -07001580 if (is_inline_node(page)) {
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001581 clear_inline_node(page);
1582 unlock_page(page);
1583 flush_inline_data(sbi, ino_of_node(page));
Chao Yu9a4cbc92016-02-22 18:35:46 +08001584 goto lock_node;
Jaegeuk Kim2049d4f2016-01-25 05:57:05 -08001585 }
1586
Jaegeuk Kimfa3d2bd2016-01-28 11:48:52 -08001587 f2fs_wait_on_page_writeback(page, NODE, true);
1588
1589 BUG_ON(PageWriteback(page));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001590 if (!clear_page_dirty_for_io(page))
1591 goto continue_unlock;
1592
Jaegeuk Kim52681372016-04-13 16:24:44 -07001593 set_fsync_mark(page, 0);
1594 set_dentry_mark(page, 0);
Jaegeuk Kim52746512014-08-11 18:18:36 -07001595
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001596 ret = __write_node_page(page, false, &submitted, wbc);
1597 if (ret)
Jaegeuk Kim52746512014-08-11 18:18:36 -07001598 unlock_page(page);
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001599 else if (submitted)
Chao Yu3f5f4952016-09-29 18:50:10 +08001600 nwritten++;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001601
1602 if (--wbc->nr_to_write == 0)
1603 break;
1604 }
1605 pagevec_release(&pvec);
1606 cond_resched();
1607
1608 if (wbc->nr_to_write == 0) {
1609 step = 2;
1610 break;
1611 }
1612 }
1613
1614 if (step < 2) {
1615 step++;
1616 goto next_step;
1617 }
Chao Yu3f5f4952016-09-29 18:50:10 +08001618out:
1619 if (nwritten)
1620 f2fs_submit_merged_bio(sbi, NODE, WRITE);
1621 return ret;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001622}
1623
Jaegeuk Kimcfe58f92013-10-31 14:57:01 +09001624int wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, nid_t ino)
1625{
Chao Yu80dd9c02016-02-24 17:20:44 +08001626 pgoff_t index = 0, end = ULONG_MAX;
Jaegeuk Kimcfe58f92013-10-31 14:57:01 +09001627 struct pagevec pvec;
Miklos Szeredi280db3c2016-09-16 12:44:21 +02001628 int ret2, ret = 0;
Jaegeuk Kimcfe58f92013-10-31 14:57:01 +09001629
1630 pagevec_init(&pvec, 0);
Jaegeuk Kim4ef51a82014-01-21 18:51:16 +09001631
1632 while (index <= end) {
1633 int i, nr_pages;
1634 nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
1635 PAGECACHE_TAG_WRITEBACK,
1636 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1637 if (nr_pages == 0)
1638 break;
Jaegeuk Kimcfe58f92013-10-31 14:57:01 +09001639
1640 for (i = 0; i < nr_pages; i++) {
1641 struct page *page = pvec.pages[i];
1642
1643 /* until radix tree lookup accepts end_index */
Chao Yucfb271d2013-12-05 17:15:22 +08001644 if (unlikely(page->index > end))
Jaegeuk Kimcfe58f92013-10-31 14:57:01 +09001645 continue;
1646
Chao Yu4bf08ff2013-11-04 10:28:33 +08001647 if (ino && ino_of_node(page) == ino) {
Jaegeuk Kimfec1d652016-01-20 23:43:51 +08001648 f2fs_wait_on_page_writeback(page, NODE, true);
Chao Yu4bf08ff2013-11-04 10:28:33 +08001649 if (TestClearPageError(page))
1650 ret = -EIO;
1651 }
Jaegeuk Kimcfe58f92013-10-31 14:57:01 +09001652 }
1653 pagevec_release(&pvec);
1654 cond_resched();
1655 }
1656
Miklos Szeredi280db3c2016-09-16 12:44:21 +02001657 ret2 = filemap_check_errors(NODE_MAPPING(sbi));
Jaegeuk Kimcfe58f92013-10-31 14:57:01 +09001658 if (!ret)
1659 ret = ret2;
1660 return ret;
1661}
1662
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001663static int f2fs_write_node_pages(struct address_space *mapping,
1664 struct writeback_control *wbc)
1665{
Jaegeuk Kim40813632014-09-02 15:31:18 -07001666 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
Jaegeuk Kim9dfa1ba2016-07-13 19:33:19 -07001667 struct blk_plug plug;
Jaegeuk Kim50c8cdb2014-03-18 13:47:11 +09001668 long diff;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001669
Jaegeuk Kim4660f9c2013-10-24 14:19:18 +09001670 /* balancing f2fs's metadata in background */
1671 f2fs_balance_fs_bg(sbi);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001672
Jaegeuk Kima7fdffb2013-01-18 14:54:13 +09001673 /* collect a number of dirty node pages and write together */
Jaegeuk Kim87d6f892014-03-18 12:40:49 +09001674 if (get_pages(sbi, F2FS_DIRTY_NODES) < nr_pages_to_skip(sbi, NODE))
Jaegeuk Kimd3baf952014-03-18 13:43:05 +09001675 goto skip_write;
Jaegeuk Kima7fdffb2013-01-18 14:54:13 +09001676
Yunlei Hed31c7c32016-02-04 16:14:00 +08001677 trace_f2fs_writepages(mapping->host, wbc, NODE);
1678
Jaegeuk Kim50c8cdb2014-03-18 13:47:11 +09001679 diff = nr_pages_to_write(sbi, NODE, wbc);
Jaegeuk Kimfb5566d2014-01-08 10:09:51 +09001680 wbc->sync_mode = WB_SYNC_NONE;
Jaegeuk Kim9dfa1ba2016-07-13 19:33:19 -07001681 blk_start_plug(&plug);
Jaegeuk Kim52681372016-04-13 16:24:44 -07001682 sync_node_pages(sbi, wbc);
Jaegeuk Kim9dfa1ba2016-07-13 19:33:19 -07001683 blk_finish_plug(&plug);
Jaegeuk Kim50c8cdb2014-03-18 13:47:11 +09001684 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001685 return 0;
Jaegeuk Kimd3baf952014-03-18 13:43:05 +09001686
1687skip_write:
1688 wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES);
Yunlei Hed31c7c32016-02-04 16:14:00 +08001689 trace_f2fs_writepages(mapping->host, wbc, NODE);
Jaegeuk Kimd3baf952014-03-18 13:43:05 +09001690 return 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001691}
1692
1693static int f2fs_set_node_page_dirty(struct page *page)
1694{
Jaegeuk Kim26c6b882013-10-24 17:53:29 +09001695 trace_f2fs_set_page_dirty(page, NODE);
1696
Jaegeuk Kim237c0792016-06-30 18:49:15 -07001697 if (!PageUptodate(page))
1698 SetPageUptodate(page);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001699 if (!PageDirty(page)) {
Jaegeuk Kimfe76b792016-06-30 18:40:10 -07001700 f2fs_set_page_dirty_nobuffers(page);
Jaegeuk Kim40813632014-09-02 15:31:18 -07001701 inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001702 SetPagePrivate(page);
Jaegeuk Kim9e4ded32014-12-17 19:58:58 -08001703 f2fs_trace_pid(page);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001704 return 1;
1705 }
1706 return 0;
1707}
1708
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +09001709/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001710 * Structure of the f2fs node operations
1711 */
1712const struct address_space_operations f2fs_node_aops = {
1713 .writepage = f2fs_write_node_page,
1714 .writepages = f2fs_write_node_pages,
1715 .set_page_dirty = f2fs_set_node_page_dirty,
Chao Yu487261f2015-02-05 17:44:29 +08001716 .invalidatepage = f2fs_invalidate_page,
1717 .releasepage = f2fs_release_page,
Weichao Guo5b7a4872016-09-20 05:03:27 +08001718#ifdef CONFIG_MIGRATION
1719 .migratepage = f2fs_migrate_page,
1720#endif
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001721};
1722
Jaegeuk Kim8a7ed662014-02-21 14:29:35 +09001723static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,
1724 nid_t n)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001725{
Jaegeuk Kim8a7ed662014-02-21 14:29:35 +09001726 return radix_tree_lookup(&nm_i->free_nid_root, n);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001727}
1728
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001729static int __insert_nid_to_list(struct f2fs_sb_info *sbi,
1730 struct free_nid *i, enum nid_list list, bool new)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001731{
Chao Yub8559dc2016-10-12 19:28:29 +08001732 struct f2fs_nm_info *nm_i = NM_I(sbi);
1733
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001734 if (new) {
1735 int err = radix_tree_insert(&nm_i->free_nid_root, i->nid, i);
1736 if (err)
1737 return err;
1738 }
1739
Chao Yub8559dc2016-10-12 19:28:29 +08001740 f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW :
1741 i->state != NID_ALLOC);
1742 nm_i->nid_cnt[list]++;
1743 list_add_tail(&i->list, &nm_i->nid_list[list]);
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001744 return 0;
Chao Yub8559dc2016-10-12 19:28:29 +08001745}
1746
1747static void __remove_nid_from_list(struct f2fs_sb_info *sbi,
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001748 struct free_nid *i, enum nid_list list, bool reuse)
Chao Yub8559dc2016-10-12 19:28:29 +08001749{
1750 struct f2fs_nm_info *nm_i = NM_I(sbi);
1751
1752 f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW :
1753 i->state != NID_ALLOC);
1754 nm_i->nid_cnt[list]--;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001755 list_del(&i->list);
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001756 if (!reuse)
1757 radix_tree_delete(&nm_i->free_nid_root, i->nid);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001758}
1759
Chao Yu4ac91242017-02-23 10:53:49 +08001760/* return if the nid is recognized as free */
1761static bool add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001762{
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +09001763 struct f2fs_nm_info *nm_i = NM_I(sbi);
Chao Yu30a61dd2017-03-22 14:45:05 +08001764 struct free_nid *i, *e;
Jaegeuk Kim59bbd472013-05-07 20:47:40 +09001765 struct nat_entry *ne;
Chao Yu30a61dd2017-03-22 14:45:05 +08001766 int err = -EINVAL;
1767 bool ret = false;
Jaegeuk Kim9198ace2013-04-25 13:21:12 +09001768
1769 /* 0 nid should not be used */
Chao Yucfb271d2013-12-05 17:15:22 +08001770 if (unlikely(nid == 0))
Chao Yu4ac91242017-02-23 10:53:49 +08001771 return false;
Jaegeuk Kim59bbd472013-05-07 20:47:40 +09001772
Gu Zheng7bd59382013-10-22 14:52:26 +08001773 i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001774 i->nid = nid;
1775 i->state = NID_NEW;
1776
Chao Yu30a61dd2017-03-22 14:45:05 +08001777 if (radix_tree_preload(GFP_NOFS))
1778 goto err;
Jaegeuk Kim769ec6e2014-12-03 20:47:26 -08001779
Chao Yub8559dc2016-10-12 19:28:29 +08001780 spin_lock(&nm_i->nid_list_lock);
Chao Yu30a61dd2017-03-22 14:45:05 +08001781
1782 if (build) {
1783 /*
1784 * Thread A Thread B
1785 * - f2fs_create
1786 * - f2fs_new_inode
1787 * - alloc_nid
1788 * - __insert_nid_to_list(ALLOC_NID_LIST)
1789 * - f2fs_balance_fs_bg
1790 * - build_free_nids
1791 * - __build_free_nids
1792 * - scan_nat_page
1793 * - add_free_nid
1794 * - __lookup_nat_cache
1795 * - f2fs_add_link
1796 * - init_inode_metadata
1797 * - new_inode_page
1798 * - new_node_page
1799 * - set_node_addr
1800 * - alloc_nid_done
1801 * - __remove_nid_from_list(ALLOC_NID_LIST)
1802 * - __insert_nid_to_list(FREE_NID_LIST)
1803 */
1804 ne = __lookup_nat_cache(nm_i, nid);
1805 if (ne && (!get_nat_flag(ne, IS_CHECKPOINTED) ||
1806 nat_get_blkaddr(ne) != NULL_ADDR))
1807 goto err_out;
1808
1809 e = __lookup_free_nid_list(nm_i, nid);
1810 if (e) {
1811 if (e->state == NID_NEW)
1812 ret = true;
1813 goto err_out;
1814 }
1815 }
1816 ret = true;
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001817 err = __insert_nid_to_list(sbi, i, FREE_NID_LIST, true);
Chao Yu30a61dd2017-03-22 14:45:05 +08001818err_out:
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001819 spin_unlock(&nm_i->nid_list_lock);
1820 radix_tree_preload_end();
Chao Yu30a61dd2017-03-22 14:45:05 +08001821err:
1822 if (err)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001823 kmem_cache_free(free_nid_slab, i);
Chao Yu30a61dd2017-03-22 14:45:05 +08001824 return ret;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001825}
1826
Chao Yub8559dc2016-10-12 19:28:29 +08001827static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001828{
Chao Yub8559dc2016-10-12 19:28:29 +08001829 struct f2fs_nm_info *nm_i = NM_I(sbi);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001830 struct free_nid *i;
Chao Yucf0ee0f2014-04-02 08:55:00 +08001831 bool need_free = false;
1832
Chao Yub8559dc2016-10-12 19:28:29 +08001833 spin_lock(&nm_i->nid_list_lock);
Jaegeuk Kim8a7ed662014-02-21 14:29:35 +09001834 i = __lookup_free_nid_list(nm_i, nid);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001835 if (i && i->state == NID_NEW) {
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07001836 __remove_nid_from_list(sbi, i, FREE_NID_LIST, false);
Chao Yucf0ee0f2014-04-02 08:55:00 +08001837 need_free = true;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001838 }
Chao Yub8559dc2016-10-12 19:28:29 +08001839 spin_unlock(&nm_i->nid_list_lock);
Chao Yucf0ee0f2014-04-02 08:55:00 +08001840
1841 if (need_free)
1842 kmem_cache_free(free_nid_slab, i);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001843}
1844
Jaegeuk Kim9f7e4a22017-03-10 09:39:57 -08001845static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid,
Chao Yu346fe752017-03-13 20:10:41 +08001846 bool set, bool build)
Chao Yu4ac91242017-02-23 10:53:49 +08001847{
1848 struct f2fs_nm_info *nm_i = NM_I(sbi);
1849 unsigned int nat_ofs = NAT_BLOCK_OFFSET(nid);
1850 unsigned int nid_ofs = nid - START_NID(nid);
1851
1852 if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap))
1853 return;
1854
1855 if (set)
Jaegeuk Kim23380b82017-03-07 14:11:06 -08001856 __set_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]);
Chao Yu4ac91242017-02-23 10:53:49 +08001857 else
Jaegeuk Kim23380b82017-03-07 14:11:06 -08001858 __clear_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]);
Chao Yu586d1492017-03-01 17:09:07 +08001859
Chao Yu586d1492017-03-01 17:09:07 +08001860 if (set)
1861 nm_i->free_nid_count[nat_ofs]++;
1862 else if (!build)
1863 nm_i->free_nid_count[nat_ofs]--;
Chao Yu4ac91242017-02-23 10:53:49 +08001864}
1865
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +09001866static void scan_nat_page(struct f2fs_sb_info *sbi,
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001867 struct page *nat_page, nid_t start_nid)
1868{
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +09001869 struct f2fs_nm_info *nm_i = NM_I(sbi);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001870 struct f2fs_nat_block *nat_blk = page_address(nat_page);
1871 block_t blk_addr;
Chao Yu4ac91242017-02-23 10:53:49 +08001872 unsigned int nat_ofs = NAT_BLOCK_OFFSET(start_nid);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001873 int i;
1874
Chao Yu586d1492017-03-01 17:09:07 +08001875 if (test_bit_le(nat_ofs, nm_i->nat_block_bitmap))
1876 return;
1877
Jaegeuk Kim23380b82017-03-07 14:11:06 -08001878 __set_bit_le(nat_ofs, nm_i->nat_block_bitmap);
Chao Yu4ac91242017-02-23 10:53:49 +08001879
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001880 i = start_nid % NAT_ENTRY_PER_BLOCK;
1881
1882 for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) {
Chao Yu4ac91242017-02-23 10:53:49 +08001883 bool freed = false;
Haicheng Li23d38842013-05-06 23:15:43 +08001884
Chao Yucfb271d2013-12-05 17:15:22 +08001885 if (unlikely(start_nid >= nm_i->max_nid))
Jaegeuk Kim04431c42013-03-16 08:34:37 +09001886 break;
Haicheng Li23d38842013-05-06 23:15:43 +08001887
1888 blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -07001889 f2fs_bug_on(sbi, blk_addr == NEW_ADDR);
Chao Yu3a2ad562016-10-11 22:31:35 +08001890 if (blk_addr == NULL_ADDR)
Chao Yu4ac91242017-02-23 10:53:49 +08001891 freed = add_free_nid(sbi, start_nid, true);
Chao Yu346fe752017-03-13 20:10:41 +08001892 spin_lock(&NM_I(sbi)->nid_list_lock);
1893 update_free_nid_bitmap(sbi, start_nid, freed, true);
1894 spin_unlock(&NM_I(sbi)->nid_list_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001895 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001896}
1897
Chao Yu4ac91242017-02-23 10:53:49 +08001898static void scan_free_nid_bits(struct f2fs_sb_info *sbi)
1899{
1900 struct f2fs_nm_info *nm_i = NM_I(sbi);
1901 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
1902 struct f2fs_journal *journal = curseg->journal;
1903 unsigned int i, idx;
Chao Yu4ac91242017-02-23 10:53:49 +08001904
1905 down_read(&nm_i->nat_tree_lock);
1906
1907 for (i = 0; i < nm_i->nat_blocks; i++) {
1908 if (!test_bit_le(i, nm_i->nat_block_bitmap))
1909 continue;
Chao Yu586d1492017-03-01 17:09:07 +08001910 if (!nm_i->free_nid_count[i])
1911 continue;
Chao Yu4ac91242017-02-23 10:53:49 +08001912 for (idx = 0; idx < NAT_ENTRY_PER_BLOCK; idx++) {
1913 nid_t nid;
1914
1915 if (!test_bit_le(idx, nm_i->free_nid_bitmap[i]))
1916 continue;
1917
1918 nid = i * NAT_ENTRY_PER_BLOCK + idx;
1919 add_free_nid(sbi, nid, true);
1920
Kinglong Meef0cdbfe2017-02-26 20:47:16 +08001921 if (nm_i->nid_cnt[FREE_NID_LIST] >= MAX_FREE_NIDS)
Chao Yu4ac91242017-02-23 10:53:49 +08001922 goto out;
1923 }
1924 }
1925out:
1926 down_read(&curseg->journal_rwsem);
1927 for (i = 0; i < nats_in_cursum(journal); i++) {
1928 block_t addr;
1929 nid_t nid;
1930
1931 addr = le32_to_cpu(nat_in_journal(journal, i).block_addr);
1932 nid = le32_to_cpu(nid_in_journal(journal, i));
1933 if (addr == NULL_ADDR)
1934 add_free_nid(sbi, nid, true);
1935 else
1936 remove_free_nid(sbi, nid);
1937 }
1938 up_read(&curseg->journal_rwsem);
1939 up_read(&nm_i->nat_tree_lock);
1940}
1941
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08001942static void __build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001943{
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001944 struct f2fs_nm_info *nm_i = NM_I(sbi);
1945 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
Chao Yub7ad7512016-02-19 18:08:46 +08001946 struct f2fs_journal *journal = curseg->journal;
Haicheng Li87609522013-05-06 23:15:42 +08001947 int i = 0;
Jaegeuk Kim55008d82013-04-25 16:05:51 +09001948 nid_t nid = nm_i->next_scan_nid;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001949
Jaegeuk Kim55008d82013-04-25 16:05:51 +09001950 /* Enough entries */
Chao Yub8559dc2016-10-12 19:28:29 +08001951 if (nm_i->nid_cnt[FREE_NID_LIST] >= NAT_ENTRY_PER_BLOCK)
Jaegeuk Kim55008d82013-04-25 16:05:51 +09001952 return;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001953
Chao Yu3a2ad562016-10-11 22:31:35 +08001954 if (!sync && !available_free_memory(sbi, FREE_NIDS))
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001955 return;
1956
Chao Yu4ac91242017-02-23 10:53:49 +08001957 if (!mount) {
1958 /* try to find free nids in free_nid_bitmap */
1959 scan_free_nid_bits(sbi);
1960
1961 if (nm_i->nid_cnt[FREE_NID_LIST])
1962 return;
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08001963 }
1964
Jaegeuk Kim55008d82013-04-25 16:05:51 +09001965 /* readahead nat pages to be scanned */
Chao Yu26879fb2015-10-12 17:05:59 +08001966 ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES,
1967 META_NAT, true);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001968
Jaegeuk Kimb873b792016-08-04 11:38:25 -07001969 down_read(&nm_i->nat_tree_lock);
Jaegeuk Kima5131192016-01-02 09:19:41 -08001970
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001971 while (1) {
1972 struct page *page = get_current_nat_page(sbi, nid);
1973
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +09001974 scan_nat_page(sbi, page, nid);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001975 f2fs_put_page(page, 1);
1976
1977 nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
Chao Yucfb271d2013-12-05 17:15:22 +08001978 if (unlikely(nid >= nm_i->max_nid))
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001979 nid = 0;
Jaegeuk Kim55008d82013-04-25 16:05:51 +09001980
Chao Yua6d494b2015-07-24 18:26:26 +08001981 if (++i >= FREE_NID_PAGES)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001982 break;
1983 }
1984
Jaegeuk Kim55008d82013-04-25 16:05:51 +09001985 /* go to the next free nat pages to find free nids abundantly */
1986 nm_i->next_scan_nid = nid;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001987
1988 /* find free nids from current sum_pages */
Chao Yub7ad7512016-02-19 18:08:46 +08001989 down_read(&curseg->journal_rwsem);
Chao Yudfc08a12016-02-14 18:50:40 +08001990 for (i = 0; i < nats_in_cursum(journal); i++) {
1991 block_t addr;
1992
1993 addr = le32_to_cpu(nat_in_journal(journal, i).block_addr);
1994 nid = le32_to_cpu(nid_in_journal(journal, i));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001995 if (addr == NULL_ADDR)
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +09001996 add_free_nid(sbi, nid, true);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001997 else
Chao Yub8559dc2016-10-12 19:28:29 +08001998 remove_free_nid(sbi, nid);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09001999 }
Chao Yub7ad7512016-02-19 18:08:46 +08002000 up_read(&curseg->journal_rwsem);
Jaegeuk Kimb873b792016-08-04 11:38:25 -07002001 up_read(&nm_i->nat_tree_lock);
Chao Yu2db23882015-10-12 17:07:33 +08002002
2003 ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid),
Chao Yuea1a29a02015-10-12 17:08:48 +08002004 nm_i->ra_nid_pages, META_NAT, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002005}
2006
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002007void build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount)
Chao Yu2411cf52016-10-11 22:31:34 +08002008{
2009 mutex_lock(&NM_I(sbi)->build_lock);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002010 __build_free_nids(sbi, sync, mount);
Chao Yu2411cf52016-10-11 22:31:34 +08002011 mutex_unlock(&NM_I(sbi)->build_lock);
2012}
2013
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002014/*
2015 * If this function returns success, caller can obtain a new nid
2016 * from second parameter of this function.
2017 * The returned nid could be used ino as well as nid when inode is created.
2018 */
2019bool alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
2020{
2021 struct f2fs_nm_info *nm_i = NM_I(sbi);
2022 struct free_nid *i = NULL;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002023retry:
Jaegeuk Kimcb789422016-04-29 16:29:22 -07002024#ifdef CONFIG_F2FS_FAULT_INJECTION
Chao Yu55523512017-02-25 11:08:28 +08002025 if (time_to_inject(sbi, FAULT_ALLOC_NID)) {
2026 f2fs_show_injection_info(FAULT_ALLOC_NID);
Jaegeuk Kimcb789422016-04-29 16:29:22 -07002027 return false;
Chao Yu55523512017-02-25 11:08:28 +08002028 }
Jaegeuk Kimcb789422016-04-29 16:29:22 -07002029#endif
Chao Yub8559dc2016-10-12 19:28:29 +08002030 spin_lock(&nm_i->nid_list_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002031
Chao Yu04d47e62016-11-17 20:53:11 +08002032 if (unlikely(nm_i->available_nids == 0)) {
2033 spin_unlock(&nm_i->nid_list_lock);
2034 return false;
2035 }
Jaegeuk Kim55008d82013-04-25 16:05:51 +09002036
2037 /* We should not use stale free nids created by build_free_nids */
Chao Yub8559dc2016-10-12 19:28:29 +08002038 if (nm_i->nid_cnt[FREE_NID_LIST] && !on_build_free_nids(nm_i)) {
2039 f2fs_bug_on(sbi, list_empty(&nm_i->nid_list[FREE_NID_LIST]));
2040 i = list_first_entry(&nm_i->nid_list[FREE_NID_LIST],
2041 struct free_nid, list);
Jaegeuk Kim55008d82013-04-25 16:05:51 +09002042 *nid = i->nid;
Chao Yub8559dc2016-10-12 19:28:29 +08002043
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002044 __remove_nid_from_list(sbi, i, FREE_NID_LIST, true);
Jaegeuk Kim55008d82013-04-25 16:05:51 +09002045 i->state = NID_ALLOC;
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002046 __insert_nid_to_list(sbi, i, ALLOC_NID_LIST, false);
Chao Yu04d47e62016-11-17 20:53:11 +08002047 nm_i->available_nids--;
Chao Yu4ac91242017-02-23 10:53:49 +08002048
Chao Yu346fe752017-03-13 20:10:41 +08002049 update_free_nid_bitmap(sbi, *nid, false, false);
Chao Yu4ac91242017-02-23 10:53:49 +08002050
Chao Yub8559dc2016-10-12 19:28:29 +08002051 spin_unlock(&nm_i->nid_list_lock);
Jaegeuk Kim55008d82013-04-25 16:05:51 +09002052 return true;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002053 }
Chao Yub8559dc2016-10-12 19:28:29 +08002054 spin_unlock(&nm_i->nid_list_lock);
Jaegeuk Kim55008d82013-04-25 16:05:51 +09002055
2056 /* Let's scan nat pages and its caches to get free nids */
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002057 build_free_nids(sbi, true, false);
Jaegeuk Kim55008d82013-04-25 16:05:51 +09002058 goto retry;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002059}
2060
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +09002061/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002062 * alloc_nid() should be called prior to this function.
2063 */
2064void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
2065{
2066 struct f2fs_nm_info *nm_i = NM_I(sbi);
2067 struct free_nid *i;
2068
Chao Yub8559dc2016-10-12 19:28:29 +08002069 spin_lock(&nm_i->nid_list_lock);
Jaegeuk Kim8a7ed662014-02-21 14:29:35 +09002070 i = __lookup_free_nid_list(nm_i, nid);
Chao Yub8559dc2016-10-12 19:28:29 +08002071 f2fs_bug_on(sbi, !i);
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002072 __remove_nid_from_list(sbi, i, ALLOC_NID_LIST, false);
Chao Yub8559dc2016-10-12 19:28:29 +08002073 spin_unlock(&nm_i->nid_list_lock);
Chao Yucf0ee0f2014-04-02 08:55:00 +08002074
2075 kmem_cache_free(free_nid_slab, i);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002076}
2077
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +09002078/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002079 * alloc_nid() should be called prior to this function.
2080 */
2081void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
2082{
Jaegeuk Kim49952fa2013-04-03 22:19:03 +09002083 struct f2fs_nm_info *nm_i = NM_I(sbi);
2084 struct free_nid *i;
Chao Yucf0ee0f2014-04-02 08:55:00 +08002085 bool need_free = false;
Jaegeuk Kim49952fa2013-04-03 22:19:03 +09002086
Jaegeuk Kim65985d92013-08-14 21:57:27 +09002087 if (!nid)
2088 return;
2089
Chao Yub8559dc2016-10-12 19:28:29 +08002090 spin_lock(&nm_i->nid_list_lock);
Jaegeuk Kim8a7ed662014-02-21 14:29:35 +09002091 i = __lookup_free_nid_list(nm_i, nid);
Chao Yub8559dc2016-10-12 19:28:29 +08002092 f2fs_bug_on(sbi, !i);
2093
Jaegeuk Kim6fb03f32014-04-16 10:47:06 +09002094 if (!available_free_memory(sbi, FREE_NIDS)) {
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002095 __remove_nid_from_list(sbi, i, ALLOC_NID_LIST, false);
Chao Yucf0ee0f2014-04-02 08:55:00 +08002096 need_free = true;
Haicheng Li95630cb2013-05-06 23:15:41 +08002097 } else {
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002098 __remove_nid_from_list(sbi, i, ALLOC_NID_LIST, true);
Haicheng Li95630cb2013-05-06 23:15:41 +08002099 i->state = NID_NEW;
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002100 __insert_nid_to_list(sbi, i, FREE_NID_LIST, false);
Haicheng Li95630cb2013-05-06 23:15:41 +08002101 }
Chao Yu04d47e62016-11-17 20:53:11 +08002102
2103 nm_i->available_nids++;
2104
Chao Yu346fe752017-03-13 20:10:41 +08002105 update_free_nid_bitmap(sbi, nid, true, false);
Chao Yu4ac91242017-02-23 10:53:49 +08002106
Chao Yub8559dc2016-10-12 19:28:29 +08002107 spin_unlock(&nm_i->nid_list_lock);
Chao Yucf0ee0f2014-04-02 08:55:00 +08002108
2109 if (need_free)
2110 kmem_cache_free(free_nid_slab, i);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002111}
2112
Chao Yu31696582015-07-28 18:33:46 +08002113int try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink)
2114{
2115 struct f2fs_nm_info *nm_i = NM_I(sbi);
2116 struct free_nid *i, *next;
2117 int nr = nr_shrink;
2118
Chao Yub8559dc2016-10-12 19:28:29 +08002119 if (nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS)
Jaegeuk Kimad4edb82016-06-16 16:41:49 -07002120 return 0;
2121
Chao Yu31696582015-07-28 18:33:46 +08002122 if (!mutex_trylock(&nm_i->build_lock))
2123 return 0;
2124
Chao Yub8559dc2016-10-12 19:28:29 +08002125 spin_lock(&nm_i->nid_list_lock);
2126 list_for_each_entry_safe(i, next, &nm_i->nid_list[FREE_NID_LIST],
2127 list) {
2128 if (nr_shrink <= 0 ||
2129 nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS)
Chao Yu31696582015-07-28 18:33:46 +08002130 break;
Chao Yub8559dc2016-10-12 19:28:29 +08002131
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002132 __remove_nid_from_list(sbi, i, FREE_NID_LIST, false);
Chao Yu31696582015-07-28 18:33:46 +08002133 kmem_cache_free(free_nid_slab, i);
2134 nr_shrink--;
Chao Yu31696582015-07-28 18:33:46 +08002135 }
Chao Yub8559dc2016-10-12 19:28:29 +08002136 spin_unlock(&nm_i->nid_list_lock);
Chao Yu31696582015-07-28 18:33:46 +08002137 mutex_unlock(&nm_i->build_lock);
2138
2139 return nr - nr_shrink;
2140}
2141
Chao Yu70cfed82014-08-02 15:26:04 +08002142void recover_inline_xattr(struct inode *inode, struct page *page)
Chao Yu28cdce02014-03-11 13:37:38 +08002143{
Chao Yu28cdce02014-03-11 13:37:38 +08002144 void *src_addr, *dst_addr;
2145 size_t inline_size;
2146 struct page *ipage;
2147 struct f2fs_inode *ri;
2148
Jaegeuk Kim40813632014-09-02 15:31:18 -07002149 ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -07002150 f2fs_bug_on(F2FS_I_SB(inode), IS_ERR(ipage));
Chao Yu28cdce02014-03-11 13:37:38 +08002151
Jaegeuk Kime3b4d432014-08-07 23:45:42 -07002152 ri = F2FS_INODE(page);
2153 if (!(ri->i_inline & F2FS_INLINE_XATTR)) {
Jaegeuk Kim91942322016-05-20 10:13:22 -07002154 clear_inode_flag(inode, FI_INLINE_XATTR);
Jaegeuk Kime3b4d432014-08-07 23:45:42 -07002155 goto update_inode;
2156 }
2157
Chao Yu28cdce02014-03-11 13:37:38 +08002158 dst_addr = inline_xattr_addr(ipage);
2159 src_addr = inline_xattr_addr(page);
2160 inline_size = inline_xattr_size(inode);
2161
Jaegeuk Kimfec1d652016-01-20 23:43:51 +08002162 f2fs_wait_on_page_writeback(ipage, NODE, true);
Chao Yu28cdce02014-03-11 13:37:38 +08002163 memcpy(dst_addr, src_addr, inline_size);
Jaegeuk Kime3b4d432014-08-07 23:45:42 -07002164update_inode:
Chao Yu28cdce02014-03-11 13:37:38 +08002165 update_inode(inode, ipage);
2166 f2fs_put_page(ipage, 1);
2167}
2168
Chao Yud2600812017-02-08 17:39:45 +08002169int recover_xattr_data(struct inode *inode, struct page *page, block_t blkaddr)
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002170{
Jaegeuk Kim40813632014-09-02 15:31:18 -07002171 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002172 nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid;
2173 nid_t new_xnid = nid_of_node(page);
2174 struct node_info ni;
Chao Yud2600812017-02-08 17:39:45 +08002175 struct page *xpage;
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002176
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002177 if (!prev_xnid)
2178 goto recover_xnid;
2179
Chao Yud2600812017-02-08 17:39:45 +08002180 /* 1: invalidate the previous xattr nid */
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002181 get_node_info(sbi, prev_xnid, &ni);
Jaegeuk Kim9850cf42014-09-02 15:52:58 -07002182 f2fs_bug_on(sbi, ni.blk_addr == NULL_ADDR);
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002183 invalidate_blocks(sbi, ni.blk_addr);
2184 dec_valid_node_count(sbi, inode);
Jaegeuk Kim479f40c2014-03-20 21:52:53 +09002185 set_node_addr(sbi, &ni, NULL_ADDR, false);
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002186
2187recover_xnid:
Chao Yud2600812017-02-08 17:39:45 +08002188 /* 2: update xattr nid in inode */
2189 remove_free_nid(sbi, new_xnid);
2190 f2fs_i_xnid_write(inode, new_xnid);
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002191 if (unlikely(!inc_valid_node_count(sbi, inode)))
Jaegeuk Kim9850cf42014-09-02 15:52:58 -07002192 f2fs_bug_on(sbi, 1);
Chao Yud2600812017-02-08 17:39:45 +08002193 update_inode_page(inode);
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002194
Chao Yud2600812017-02-08 17:39:45 +08002195 /* 3: update and set xattr node page dirty */
2196 xpage = grab_cache_page(NODE_MAPPING(sbi), new_xnid);
2197 if (!xpage)
2198 return -ENOMEM;
2199
2200 memcpy(F2FS_NODE(xpage), F2FS_NODE(page), PAGE_SIZE);
2201
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002202 get_node_info(sbi, new_xnid, &ni);
2203 ni.ino = inode->i_ino;
Jaegeuk Kim479f40c2014-03-20 21:52:53 +09002204 set_node_addr(sbi, &ni, NEW_ADDR, false);
Chao Yud2600812017-02-08 17:39:45 +08002205 set_page_dirty(xpage);
2206 f2fs_put_page(xpage, 1);
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002207
Chao Yud2600812017-02-08 17:39:45 +08002208 return 0;
Jaegeuk Kimabb23662014-01-28 12:25:06 +09002209}
2210
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002211int recover_inode_page(struct f2fs_sb_info *sbi, struct page *page)
2212{
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +09002213 struct f2fs_inode *src, *dst;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002214 nid_t ino = ino_of_node(page);
2215 struct node_info old_ni, new_ni;
2216 struct page *ipage;
2217
Jaegeuk Kime8271fa2014-04-18 15:21:04 +09002218 get_node_info(sbi, ino, &old_ni);
2219
2220 if (unlikely(old_ni.blk_addr != NULL_ADDR))
2221 return -EINVAL;
Jaegeuk Kime8ea9b32016-09-09 16:59:39 -07002222retry:
Jaegeuk Kim300e1292016-04-29 16:11:53 -07002223 ipage = f2fs_grab_cache_page(NODE_MAPPING(sbi), ino, false);
Jaegeuk Kime8ea9b32016-09-09 16:59:39 -07002224 if (!ipage) {
2225 congestion_wait(BLK_RW_ASYNC, HZ/50);
2226 goto retry;
2227 }
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002228
arter97e1c42042014-08-06 23:22:50 +09002229 /* Should not use this inode from free nid list */
Chao Yub8559dc2016-10-12 19:28:29 +08002230 remove_free_nid(sbi, ino);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002231
Jaegeuk Kim237c0792016-06-30 18:49:15 -07002232 if (!PageUptodate(ipage))
2233 SetPageUptodate(ipage);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002234 fill_node_footer(ipage, ino, ino, 0, true);
2235
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +09002236 src = F2FS_INODE(page);
2237 dst = F2FS_INODE(ipage);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002238
Jaegeuk Kim58bfaf42013-12-26 16:30:41 +09002239 memcpy(dst, src, (unsigned long)&src->i_ext - (unsigned long)src);
2240 dst->i_size = 0;
2241 dst->i_blocks = cpu_to_le64(1);
2242 dst->i_links = cpu_to_le32(1);
2243 dst->i_xattr_nid = 0;
Jaegeuk Kim617deb82014-08-07 17:04:24 -07002244 dst->i_inline = src->i_inline & F2FS_INLINE_XATTR;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002245
2246 new_ni = old_ni;
2247 new_ni.ino = ino;
2248
Chao Yucfb271d2013-12-05 17:15:22 +08002249 if (unlikely(!inc_valid_node_count(sbi, NULL)))
Jaegeuk Kim65e5cd02013-05-14 15:47:43 +09002250 WARN_ON(1);
Jaegeuk Kim479f40c2014-03-20 21:52:53 +09002251 set_node_addr(sbi, &new_ni, NEW_ADDR, false);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002252 inc_valid_inode_count(sbi);
Jaegeuk Kim617deb82014-08-07 17:04:24 -07002253 set_page_dirty(ipage);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002254 f2fs_put_page(ipage, 1);
2255 return 0;
2256}
2257
2258int restore_node_summary(struct f2fs_sb_info *sbi,
2259 unsigned int segno, struct f2fs_summary_block *sum)
2260{
2261 struct f2fs_node *rn;
2262 struct f2fs_summary *sum_entry;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002263 block_t addr;
Chao Yu9ecf4b82014-12-18 18:29:05 +08002264 int i, idx, last_offset, nrpages;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002265
2266 /* scan the node segment */
2267 last_offset = sbi->blocks_per_seg;
2268 addr = START_BLOCK(sbi, segno);
2269 sum_entry = &sum->entries[0];
2270
Chao Yu9ecf4b82014-12-18 18:29:05 +08002271 for (i = 0; i < last_offset; i += nrpages, addr += nrpages) {
Jaegeuk Kim664ba972016-10-18 11:07:45 -07002272 nrpages = min(last_offset - i, BIO_MAX_PAGES);
Jaegeuk Kim393ff912013-03-08 21:29:23 +09002273
arter97e1c42042014-08-06 23:22:50 +09002274 /* readahead node pages */
Chao Yu26879fb2015-10-12 17:05:59 +08002275 ra_meta_pages(sbi, addr, nrpages, META_POR, true);
Jaegeuk Kim393ff912013-03-08 21:29:23 +09002276
Chao Yu9ecf4b82014-12-18 18:29:05 +08002277 for (idx = addr; idx < addr + nrpages; idx++) {
Chao Yu2b947002015-10-12 17:04:21 +08002278 struct page *page = get_tmp_page(sbi, idx);
Chao Yu9af0ff12013-11-22 15:48:54 +08002279
Chao Yu9ecf4b82014-12-18 18:29:05 +08002280 rn = F2FS_NODE(page);
2281 sum_entry->nid = rn->footer.nid;
2282 sum_entry->version = 0;
2283 sum_entry->ofs_in_node = 0;
2284 sum_entry++;
2285 f2fs_put_page(page, 1);
Chao Yu9af0ff12013-11-22 15:48:54 +08002286 }
Chao Yubac4eef2014-05-27 08:41:07 +08002287
Chao Yu9ecf4b82014-12-18 18:29:05 +08002288 invalidate_mapping_pages(META_MAPPING(sbi), addr,
Chao Yubac4eef2014-05-27 08:41:07 +08002289 addr + nrpages);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002290 }
Chao Yu9ecf4b82014-12-18 18:29:05 +08002291 return 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002292}
2293
Chao Yuaec71382014-06-24 09:18:20 +08002294static void remove_nats_in_journal(struct f2fs_sb_info *sbi)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002295{
2296 struct f2fs_nm_info *nm_i = NM_I(sbi);
2297 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
Chao Yub7ad7512016-02-19 18:08:46 +08002298 struct f2fs_journal *journal = curseg->journal;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002299 int i;
2300
Chao Yub7ad7512016-02-19 18:08:46 +08002301 down_write(&curseg->journal_rwsem);
Chao Yudfc08a12016-02-14 18:50:40 +08002302 for (i = 0; i < nats_in_cursum(journal); i++) {
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002303 struct nat_entry *ne;
2304 struct f2fs_nat_entry raw_ne;
Chao Yudfc08a12016-02-14 18:50:40 +08002305 nid_t nid = le32_to_cpu(nid_in_journal(journal, i));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002306
Chao Yudfc08a12016-02-14 18:50:40 +08002307 raw_ne = nat_in_journal(journal, i);
Jaegeuk Kim9be32d72014-12-05 10:39:49 -08002308
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002309 ne = __lookup_nat_cache(nm_i, nid);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002310 if (!ne) {
Chao Yu5c9e41842016-12-13 18:54:59 +08002311 ne = grab_nat_entry(nm_i, nid, true);
Jaegeuk Kim9be32d72014-12-05 10:39:49 -08002312 node_info_from_raw_nat(&ne->ni, &raw_ne);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002313 }
Chao Yu04d47e62016-11-17 20:53:11 +08002314
2315 /*
2316 * if a free nat in journal has not been used after last
2317 * checkpoint, we should remove it from available nids,
2318 * since later we will add it again.
2319 */
2320 if (!get_nat_flag(ne, IS_DIRTY) &&
2321 le32_to_cpu(raw_ne.block_addr) == NULL_ADDR) {
2322 spin_lock(&nm_i->nid_list_lock);
2323 nm_i->available_nids--;
2324 spin_unlock(&nm_i->nid_list_lock);
2325 }
2326
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002327 __set_nat_cache_dirty(nm_i, ne);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002328 }
Chao Yudfc08a12016-02-14 18:50:40 +08002329 update_nats_in_cursum(journal, -i);
Chao Yub7ad7512016-02-19 18:08:46 +08002330 up_write(&curseg->journal_rwsem);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002331}
2332
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002333static void __adjust_nat_entry_set(struct nat_entry_set *nes,
2334 struct list_head *head, int max)
2335{
2336 struct nat_entry_set *cur;
2337
2338 if (nes->entry_cnt >= max)
2339 goto add_out;
2340
2341 list_for_each_entry(cur, head, set_list) {
2342 if (cur->entry_cnt >= nes->entry_cnt) {
2343 list_add(&nes->set_list, cur->set_list.prev);
2344 return;
2345 }
2346 }
2347add_out:
2348 list_add_tail(&nes->set_list, head);
2349}
2350
Jaegeuk Kim9f7e4a22017-03-10 09:39:57 -08002351static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid,
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002352 struct page *page)
2353{
2354 struct f2fs_nm_info *nm_i = NM_I(sbi);
2355 unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK;
2356 struct f2fs_nat_block *nat_blk = page_address(page);
2357 int valid = 0;
2358 int i;
2359
2360 if (!enabled_nat_bits(sbi, NULL))
2361 return;
2362
2363 for (i = 0; i < NAT_ENTRY_PER_BLOCK; i++) {
2364 if (start_nid == 0 && i == 0)
2365 valid++;
2366 if (nat_blk->entries[i].block_addr)
2367 valid++;
2368 }
2369 if (valid == 0) {
Jaegeuk Kim23380b82017-03-07 14:11:06 -08002370 __set_bit_le(nat_index, nm_i->empty_nat_bits);
2371 __clear_bit_le(nat_index, nm_i->full_nat_bits);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002372 return;
2373 }
2374
Jaegeuk Kim23380b82017-03-07 14:11:06 -08002375 __clear_bit_le(nat_index, nm_i->empty_nat_bits);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002376 if (valid == NAT_ENTRY_PER_BLOCK)
Jaegeuk Kim23380b82017-03-07 14:11:06 -08002377 __set_bit_le(nat_index, nm_i->full_nat_bits);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002378 else
Jaegeuk Kim23380b82017-03-07 14:11:06 -08002379 __clear_bit_le(nat_index, nm_i->full_nat_bits);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002380}
2381
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002382static void __flush_nat_entry_set(struct f2fs_sb_info *sbi,
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002383 struct nat_entry_set *set, struct cp_control *cpc)
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002384{
2385 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
Chao Yub7ad7512016-02-19 18:08:46 +08002386 struct f2fs_journal *journal = curseg->journal;
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002387 nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;
2388 bool to_journal = true;
2389 struct f2fs_nat_block *nat_blk;
2390 struct nat_entry *ne, *cur;
2391 struct page *page = NULL;
2392
2393 /*
2394 * there are two steps to flush nat entries:
2395 * #1, flush nat entries to journal in current hot data summary block.
2396 * #2, flush nat entries to nat page.
2397 */
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002398 if (enabled_nat_bits(sbi, cpc) ||
2399 !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL))
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002400 to_journal = false;
2401
2402 if (to_journal) {
Chao Yub7ad7512016-02-19 18:08:46 +08002403 down_write(&curseg->journal_rwsem);
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002404 } else {
2405 page = get_next_nat_page(sbi, start_nid);
2406 nat_blk = page_address(page);
2407 f2fs_bug_on(sbi, !nat_blk);
2408 }
2409
2410 /* flush dirty nats in nat entry set */
2411 list_for_each_entry_safe(ne, cur, &set->entry_list, list) {
2412 struct f2fs_nat_entry *raw_ne;
2413 nid_t nid = nat_get_nid(ne);
2414 int offset;
2415
2416 if (nat_get_blkaddr(ne) == NEW_ADDR)
2417 continue;
2418
2419 if (to_journal) {
Chao Yudfc08a12016-02-14 18:50:40 +08002420 offset = lookup_journal_in_cursum(journal,
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002421 NAT_JOURNAL, nid, 1);
2422 f2fs_bug_on(sbi, offset < 0);
Chao Yudfc08a12016-02-14 18:50:40 +08002423 raw_ne = &nat_in_journal(journal, offset);
2424 nid_in_journal(journal, offset) = cpu_to_le32(nid);
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002425 } else {
2426 raw_ne = &nat_blk->entries[nid - start_nid];
2427 }
2428 raw_nat_from_node_info(raw_ne, &ne->ni);
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002429 nat_reset_flag(ne);
Kinglong Mee0b28b712017-02-28 21:34:47 +08002430 __clear_nat_cache_dirty(NM_I(sbi), set, ne);
Chao Yu04d47e62016-11-17 20:53:11 +08002431 if (nat_get_blkaddr(ne) == NULL_ADDR) {
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002432 add_free_nid(sbi, nid, false);
Chao Yu04d47e62016-11-17 20:53:11 +08002433 spin_lock(&NM_I(sbi)->nid_list_lock);
2434 NM_I(sbi)->available_nids++;
Chao Yu346fe752017-03-13 20:10:41 +08002435 update_free_nid_bitmap(sbi, nid, true, false);
Chao Yu4ac91242017-02-23 10:53:49 +08002436 spin_unlock(&NM_I(sbi)->nid_list_lock);
2437 } else {
2438 spin_lock(&NM_I(sbi)->nid_list_lock);
Chao Yu346fe752017-03-13 20:10:41 +08002439 update_free_nid_bitmap(sbi, nid, false, false);
Chao Yu04d47e62016-11-17 20:53:11 +08002440 spin_unlock(&NM_I(sbi)->nid_list_lock);
2441 }
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002442 }
2443
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002444 if (to_journal) {
Chao Yub7ad7512016-02-19 18:08:46 +08002445 up_write(&curseg->journal_rwsem);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002446 } else {
2447 __update_nat_bits(sbi, start_nid, page);
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002448 f2fs_put_page(page, 1);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002449 }
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002450
Yunlei He59c90812017-03-13 20:22:18 +08002451 /* Allow dirty nats by node block allocation in write_begin */
2452 if (!set->entry_cnt) {
2453 radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set);
2454 kmem_cache_free(nat_entry_set_slab, set);
2455 }
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002456}
2457
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +09002458/*
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002459 * This function is called during the checkpointing process.
2460 */
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002461void flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002462{
2463 struct f2fs_nm_info *nm_i = NM_I(sbi);
2464 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
Chao Yub7ad7512016-02-19 18:08:46 +08002465 struct f2fs_journal *journal = curseg->journal;
Jaegeuk Kim7aed0d42015-01-07 10:47:57 -08002466 struct nat_entry_set *setvec[SETVEC_SIZE];
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002467 struct nat_entry_set *set, *tmp;
2468 unsigned int found;
2469 nid_t set_idx = 0;
2470 LIST_HEAD(sets);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002471
Changman Lee20d047c2014-11-25 12:44:23 +09002472 if (!nm_i->dirty_nat_cnt)
2473 return;
Jaegeuk Kima5131192016-01-02 09:19:41 -08002474
Jaegeuk Kimb873b792016-08-04 11:38:25 -07002475 down_write(&nm_i->nat_tree_lock);
Jaegeuk Kima5131192016-01-02 09:19:41 -08002476
Chao Yuaec71382014-06-24 09:18:20 +08002477 /*
2478 * if there are no enough space in journal to store dirty nat
2479 * entries, remove all entries from journal and merge them
2480 * into nat entry set.
2481 */
Jaegeuk Kim900f7362017-02-27 21:28:53 -08002482 if (enabled_nat_bits(sbi, cpc) ||
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002483 !__has_cursum_space(journal, nm_i->dirty_nat_cnt, NAT_JOURNAL))
Chao Yuaec71382014-06-24 09:18:20 +08002484 remove_nats_in_journal(sbi);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002485
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002486 while ((found = __gang_lookup_nat_set(nm_i,
Jaegeuk Kim7aed0d42015-01-07 10:47:57 -08002487 set_idx, SETVEC_SIZE, setvec))) {
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002488 unsigned idx;
2489 set_idx = setvec[found - 1]->set + 1;
2490 for (idx = 0; idx < found; idx++)
2491 __adjust_nat_entry_set(setvec[idx], &sets,
Chao Yudfc08a12016-02-14 18:50:40 +08002492 MAX_NAT_JENTRIES(journal));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002493 }
Chao Yuaec71382014-06-24 09:18:20 +08002494
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002495 /* flush dirty nats in nat entry set */
2496 list_for_each_entry_safe(set, tmp, &sets, set_list)
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002497 __flush_nat_entry_set(sbi, set, cpc);
Jaegeuk Kim309cc2b2014-09-22 11:40:48 -07002498
Jaegeuk Kimb873b792016-08-04 11:38:25 -07002499 up_write(&nm_i->nat_tree_lock);
Yunlei He59c90812017-03-13 20:22:18 +08002500 /* Allow dirty nats by node block allocation in write_begin */
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002501}
2502
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002503static int __get_nat_bitmaps(struct f2fs_sb_info *sbi)
2504{
2505 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2506 struct f2fs_nm_info *nm_i = NM_I(sbi);
2507 unsigned int nat_bits_bytes = nm_i->nat_blocks / BITS_PER_BYTE;
2508 unsigned int i;
2509 __u64 cp_ver = cur_cp_version(ckpt);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002510 block_t nat_bits_addr;
2511
2512 if (!enabled_nat_bits(sbi, NULL))
2513 return 0;
2514
2515 nm_i->nat_bits_blocks = F2FS_BYTES_TO_BLK((nat_bits_bytes << 1) + 8 +
2516 F2FS_BLKSIZE - 1);
2517 nm_i->nat_bits = kzalloc(nm_i->nat_bits_blocks << F2FS_BLKSIZE_BITS,
2518 GFP_KERNEL);
2519 if (!nm_i->nat_bits)
2520 return -ENOMEM;
2521
2522 nat_bits_addr = __start_cp_addr(sbi) + sbi->blocks_per_seg -
2523 nm_i->nat_bits_blocks;
2524 for (i = 0; i < nm_i->nat_bits_blocks; i++) {
2525 struct page *page = get_meta_page(sbi, nat_bits_addr++);
2526
2527 memcpy(nm_i->nat_bits + (i << F2FS_BLKSIZE_BITS),
2528 page_address(page), F2FS_BLKSIZE);
2529 f2fs_put_page(page, 1);
2530 }
2531
Kinglong Meeced2c7e2017-02-25 19:53:39 +08002532 cp_ver |= (cur_cp_crc(ckpt) << 32);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002533 if (cpu_to_le64(cp_ver) != *(__le64 *)nm_i->nat_bits) {
2534 disable_nat_bits(sbi, true);
2535 return 0;
2536 }
2537
2538 nm_i->full_nat_bits = nm_i->nat_bits + 8;
2539 nm_i->empty_nat_bits = nm_i->full_nat_bits + nat_bits_bytes;
2540
2541 f2fs_msg(sbi->sb, KERN_NOTICE, "Found nat_bits in checkpoint");
2542 return 0;
2543}
2544
Chao Yu7041d5d2017-03-08 20:07:49 +08002545inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi)
2546{
2547 struct f2fs_nm_info *nm_i = NM_I(sbi);
2548 unsigned int i = 0;
2549 nid_t nid, last_nid;
2550
2551 if (!enabled_nat_bits(sbi, NULL))
2552 return;
2553
2554 for (i = 0; i < nm_i->nat_blocks; i++) {
2555 i = find_next_bit_le(nm_i->empty_nat_bits, nm_i->nat_blocks, i);
2556 if (i >= nm_i->nat_blocks)
2557 break;
2558
2559 __set_bit_le(i, nm_i->nat_block_bitmap);
2560
2561 nid = i * NAT_ENTRY_PER_BLOCK;
2562 last_nid = (i + 1) * NAT_ENTRY_PER_BLOCK;
2563
Chao Yu346fe752017-03-13 20:10:41 +08002564 spin_lock(&NM_I(sbi)->nid_list_lock);
Chao Yu7041d5d2017-03-08 20:07:49 +08002565 for (; nid < last_nid; nid++)
Chao Yu346fe752017-03-13 20:10:41 +08002566 update_free_nid_bitmap(sbi, nid, true, true);
2567 spin_unlock(&NM_I(sbi)->nid_list_lock);
Chao Yu7041d5d2017-03-08 20:07:49 +08002568 }
2569
2570 for (i = 0; i < nm_i->nat_blocks; i++) {
2571 i = find_next_bit_le(nm_i->full_nat_bits, nm_i->nat_blocks, i);
2572 if (i >= nm_i->nat_blocks)
2573 break;
2574
2575 __set_bit_le(i, nm_i->nat_block_bitmap);
2576 }
2577}
2578
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002579static int init_node_manager(struct f2fs_sb_info *sbi)
2580{
2581 struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi);
2582 struct f2fs_nm_info *nm_i = NM_I(sbi);
2583 unsigned char *version_bitmap;
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002584 unsigned int nat_segs;
2585 int err;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002586
2587 nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);
2588
2589 /* segment_count_nat includes pair segment so divide to 2. */
2590 nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1;
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002591 nm_i->nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg);
2592 nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nm_i->nat_blocks;
Jaegeuk Kim7ee0eea2014-04-18 11:14:37 +09002593
Jaegeuk Kimb63da152014-02-17 12:44:20 +09002594 /* not used nids: 0, node, meta, (and root counted as valid node) */
Chao Yu04d47e62016-11-17 20:53:11 +08002595 nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count -
2596 F2FS_RESERVED_NODE_NUM;
Chao Yub8559dc2016-10-12 19:28:29 +08002597 nm_i->nid_cnt[FREE_NID_LIST] = 0;
2598 nm_i->nid_cnt[ALLOC_NID_LIST] = 0;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002599 nm_i->nat_cnt = 0;
Jaegeuk Kimcdfc41c2014-03-19 13:31:37 +09002600 nm_i->ram_thresh = DEF_RAM_THRESHOLD;
Chao Yuea1a29a02015-10-12 17:08:48 +08002601 nm_i->ra_nid_pages = DEF_RA_NID_PAGES;
Chao Yu2304cb02016-01-18 18:32:58 +08002602 nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002603
Jaegeuk Kim8a7ed662014-02-21 14:29:35 +09002604 INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);
Chao Yub8559dc2016-10-12 19:28:29 +08002605 INIT_LIST_HEAD(&nm_i->nid_list[FREE_NID_LIST]);
2606 INIT_LIST_HEAD(&nm_i->nid_list[ALLOC_NID_LIST]);
Jaegeuk Kim769ec6e2014-12-03 20:47:26 -08002607 INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
2608 INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002609 INIT_LIST_HEAD(&nm_i->nat_entries);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002610
2611 mutex_init(&nm_i->build_lock);
Chao Yub8559dc2016-10-12 19:28:29 +08002612 spin_lock_init(&nm_i->nid_list_lock);
Jaegeuk Kimb873b792016-08-04 11:38:25 -07002613 init_rwsem(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002614
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002615 nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
Alexandru Gheorghiu79b57932013-03-28 02:24:53 +02002616 nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002617 version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
2618 if (!version_bitmap)
2619 return -EFAULT;
2620
Alexandru Gheorghiu79b57932013-03-28 02:24:53 +02002621 nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
2622 GFP_KERNEL);
2623 if (!nm_i->nat_bitmap)
2624 return -ENOMEM;
Chao Yu599a09b2017-01-07 18:52:01 +08002625
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002626 err = __get_nat_bitmaps(sbi);
2627 if (err)
2628 return err;
2629
Chao Yu599a09b2017-01-07 18:52:01 +08002630#ifdef CONFIG_F2FS_CHECK_FS
2631 nm_i->nat_bitmap_mir = kmemdup(version_bitmap, nm_i->bitmap_size,
2632 GFP_KERNEL);
2633 if (!nm_i->nat_bitmap_mir)
2634 return -ENOMEM;
2635#endif
2636
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002637 return 0;
2638}
2639
Jaegeuk Kim9f7e4a22017-03-10 09:39:57 -08002640static int init_free_nid_cache(struct f2fs_sb_info *sbi)
Chao Yu4ac91242017-02-23 10:53:49 +08002641{
2642 struct f2fs_nm_info *nm_i = NM_I(sbi);
2643
2644 nm_i->free_nid_bitmap = f2fs_kvzalloc(nm_i->nat_blocks *
2645 NAT_ENTRY_BITMAP_SIZE, GFP_KERNEL);
2646 if (!nm_i->free_nid_bitmap)
2647 return -ENOMEM;
2648
2649 nm_i->nat_block_bitmap = f2fs_kvzalloc(nm_i->nat_blocks / 8,
2650 GFP_KERNEL);
2651 if (!nm_i->nat_block_bitmap)
2652 return -ENOMEM;
Chao Yu586d1492017-03-01 17:09:07 +08002653
2654 nm_i->free_nid_count = f2fs_kvzalloc(nm_i->nat_blocks *
2655 sizeof(unsigned short), GFP_KERNEL);
2656 if (!nm_i->free_nid_count)
2657 return -ENOMEM;
Chao Yu4ac91242017-02-23 10:53:49 +08002658 return 0;
2659}
2660
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002661int build_node_manager(struct f2fs_sb_info *sbi)
2662{
2663 int err;
2664
2665 sbi->nm_info = kzalloc(sizeof(struct f2fs_nm_info), GFP_KERNEL);
2666 if (!sbi->nm_info)
2667 return -ENOMEM;
2668
2669 err = init_node_manager(sbi);
2670 if (err)
2671 return err;
2672
Chao Yu4ac91242017-02-23 10:53:49 +08002673 err = init_free_nid_cache(sbi);
2674 if (err)
2675 return err;
2676
Chao Yu7041d5d2017-03-08 20:07:49 +08002677 /* load free nid status from nat_bits table */
2678 load_free_nid_bitmap(sbi);
2679
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002680 build_free_nids(sbi, true, true);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002681 return 0;
2682}
2683
2684void destroy_node_manager(struct f2fs_sb_info *sbi)
2685{
2686 struct f2fs_nm_info *nm_i = NM_I(sbi);
2687 struct free_nid *i, *next_i;
2688 struct nat_entry *natvec[NATVEC_SIZE];
Jaegeuk Kim7aed0d42015-01-07 10:47:57 -08002689 struct nat_entry_set *setvec[SETVEC_SIZE];
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002690 nid_t nid = 0;
2691 unsigned int found;
2692
2693 if (!nm_i)
2694 return;
2695
2696 /* destroy free nid list */
Chao Yub8559dc2016-10-12 19:28:29 +08002697 spin_lock(&nm_i->nid_list_lock);
2698 list_for_each_entry_safe(i, next_i, &nm_i->nid_list[FREE_NID_LIST],
2699 list) {
Jaegeuk Kimeb0aa4b2016-10-12 10:09:59 -07002700 __remove_nid_from_list(sbi, i, FREE_NID_LIST, false);
Chao Yub8559dc2016-10-12 19:28:29 +08002701 spin_unlock(&nm_i->nid_list_lock);
Chao Yucf0ee0f2014-04-02 08:55:00 +08002702 kmem_cache_free(free_nid_slab, i);
Chao Yub8559dc2016-10-12 19:28:29 +08002703 spin_lock(&nm_i->nid_list_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002704 }
Chao Yub8559dc2016-10-12 19:28:29 +08002705 f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID_LIST]);
2706 f2fs_bug_on(sbi, nm_i->nid_cnt[ALLOC_NID_LIST]);
2707 f2fs_bug_on(sbi, !list_empty(&nm_i->nid_list[ALLOC_NID_LIST]));
2708 spin_unlock(&nm_i->nid_list_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002709
2710 /* destroy nat cache */
Jaegeuk Kimb873b792016-08-04 11:38:25 -07002711 down_write(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002712 while ((found = __gang_lookup_nat_cache(nm_i,
2713 nid, NATVEC_SIZE, natvec))) {
2714 unsigned idx;
Jaegeuk Kim7aed0d42015-01-07 10:47:57 -08002715
Gu Zhengb6ce3912014-03-07 18:43:24 +08002716 nid = nat_get_nid(natvec[found - 1]) + 1;
2717 for (idx = 0; idx < found; idx++)
2718 __del_from_nat_cache(nm_i, natvec[idx]);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002719 }
Jaegeuk Kim9850cf42014-09-02 15:52:58 -07002720 f2fs_bug_on(sbi, nm_i->nat_cnt);
Jaegeuk Kim7aed0d42015-01-07 10:47:57 -08002721
2722 /* destroy nat set cache */
2723 nid = 0;
2724 while ((found = __gang_lookup_nat_set(nm_i,
2725 nid, SETVEC_SIZE, setvec))) {
2726 unsigned idx;
2727
2728 nid = setvec[found - 1]->set + 1;
2729 for (idx = 0; idx < found; idx++) {
2730 /* entry_cnt is not zero, when cp_error was occurred */
2731 f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list));
2732 radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set);
2733 kmem_cache_free(nat_entry_set_slab, setvec[idx]);
2734 }
2735 }
Jaegeuk Kimb873b792016-08-04 11:38:25 -07002736 up_write(&nm_i->nat_tree_lock);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002737
Chao Yu4ac91242017-02-23 10:53:49 +08002738 kvfree(nm_i->nat_block_bitmap);
2739 kvfree(nm_i->free_nid_bitmap);
Chao Yu586d1492017-03-01 17:09:07 +08002740 kvfree(nm_i->free_nid_count);
Chao Yu4ac91242017-02-23 10:53:49 +08002741
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002742 kfree(nm_i->nat_bitmap);
Jaegeuk Kim22ad0b62017-02-09 10:38:09 -08002743 kfree(nm_i->nat_bits);
Chao Yu599a09b2017-01-07 18:52:01 +08002744#ifdef CONFIG_F2FS_CHECK_FS
2745 kfree(nm_i->nat_bitmap_mir);
2746#endif
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002747 sbi->nm_info = NULL;
2748 kfree(nm_i);
2749}
2750
Namjae Jeon6e6093a2013-01-17 00:08:30 +09002751int __init create_node_manager_caches(void)
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002752{
2753 nat_entry_slab = f2fs_kmem_cache_create("nat_entry",
Gu Zhenge8512d22014-03-07 18:43:28 +08002754 sizeof(struct nat_entry));
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002755 if (!nat_entry_slab)
Chao Yuaec71382014-06-24 09:18:20 +08002756 goto fail;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002757
2758 free_nid_slab = f2fs_kmem_cache_create("free_nid",
Gu Zhenge8512d22014-03-07 18:43:28 +08002759 sizeof(struct free_nid));
Chao Yuaec71382014-06-24 09:18:20 +08002760 if (!free_nid_slab)
Markus Elfringce3e6d22014-11-24 15:52:00 +01002761 goto destroy_nat_entry;
Chao Yuaec71382014-06-24 09:18:20 +08002762
2763 nat_entry_set_slab = f2fs_kmem_cache_create("nat_entry_set",
2764 sizeof(struct nat_entry_set));
2765 if (!nat_entry_set_slab)
Markus Elfringce3e6d22014-11-24 15:52:00 +01002766 goto destroy_free_nid;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002767 return 0;
Chao Yuaec71382014-06-24 09:18:20 +08002768
Markus Elfringce3e6d22014-11-24 15:52:00 +01002769destroy_free_nid:
Chao Yuaec71382014-06-24 09:18:20 +08002770 kmem_cache_destroy(free_nid_slab);
Markus Elfringce3e6d22014-11-24 15:52:00 +01002771destroy_nat_entry:
Chao Yuaec71382014-06-24 09:18:20 +08002772 kmem_cache_destroy(nat_entry_slab);
2773fail:
2774 return -ENOMEM;
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002775}
2776
2777void destroy_node_manager_caches(void)
2778{
Chao Yuaec71382014-06-24 09:18:20 +08002779 kmem_cache_destroy(nat_entry_set_slab);
Jaegeuk Kime05df3b2012-11-02 17:08:50 +09002780 kmem_cache_destroy(free_nid_slab);
2781 kmem_cache_destroy(nat_entry_slab);
2782}