blob: 9ee3bd705b944baca85818e66c4eec3385277223 [file] [log] [blame]
Ricardo Labiaga85e174b2010-10-20 00:17:58 -04001/*
2 * pNFS functions to call and manage layout drivers.
3 *
4 * Copyright (c) 2002 [year of first publication]
5 * The Regents of the University of Michigan
6 * All Rights Reserved
7 *
8 * Dean Hildebrand <dhildebz@umich.edu>
9 *
10 * Permission is granted to use, copy, create derivative works, and
11 * redistribute this software and such derivative works for any purpose,
12 * so long as the name of the University of Michigan is not used in
13 * any advertising or publicity pertaining to the use or distribution
14 * of this software without specific, written prior authorization. If
15 * the above copyright notice or any other identification of the
16 * University of Michigan is included in any copy of any portion of
17 * this software, then the disclaimer below must also be included.
18 *
19 * This software is provided as is, without representation or warranty
20 * of any kind either express or implied, including without limitation
21 * the implied warranties of merchantability, fitness for a particular
22 * purpose, or noninfringement. The Regents of the University of
23 * Michigan shall not be liable for any damages, including special,
24 * indirect, incidental, or consequential damages, with respect to any
25 * claim arising out of or in connection with the use of the software,
26 * even if it has been or is hereafter advised of the possibility of
27 * such damages.
28 */
29
30#include <linux/nfs_fs.h>
Trond Myklebust493292d2011-07-13 15:58:28 -040031#include <linux/nfs_page.h>
Paul Gortmaker143cb492011-07-01 14:23:34 -040032#include <linux/module.h>
Andy Adamson974cec82010-10-20 00:18:02 -040033#include "internal.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040034#include "pnfs.h"
Andy Adamson64419a92011-03-01 01:34:16 +000035#include "iostat.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040036
37#define NFSDBG_FACILITY NFSDBG_PNFS
Trond Myklebust25c75332012-09-18 17:01:12 -040038#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040039
Fred Isaman02c35fc2010-10-20 00:17:59 -040040/* Locking:
41 *
42 * pnfs_spinlock:
43 * protects pnfs_modules_tbl.
44 */
45static DEFINE_SPINLOCK(pnfs_spinlock);
46
47/*
48 * pnfs_modules_tbl holds all pnfs modules
49 */
50static LIST_HEAD(pnfs_modules_tbl);
51
52/* Return the registered pnfs layout driver module matching given id */
53static struct pnfs_layoutdriver_type *
54find_pnfs_driver_locked(u32 id)
55{
56 struct pnfs_layoutdriver_type *local;
57
58 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
59 if (local->id == id)
60 goto out;
61 local = NULL;
62out:
63 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
64 return local;
65}
66
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040067static struct pnfs_layoutdriver_type *
68find_pnfs_driver(u32 id)
69{
Fred Isaman02c35fc2010-10-20 00:17:59 -040070 struct pnfs_layoutdriver_type *local;
71
72 spin_lock(&pnfs_spinlock);
73 local = find_pnfs_driver_locked(id);
Trond Myklebust0a9c63f2012-06-15 13:02:58 -040074 if (local != NULL && !try_module_get(local->owner)) {
75 dprintk("%s: Could not grab reference on module\n", __func__);
76 local = NULL;
77 }
Fred Isaman02c35fc2010-10-20 00:17:59 -040078 spin_unlock(&pnfs_spinlock);
79 return local;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040080}
81
82void
83unset_pnfs_layoutdriver(struct nfs_server *nfss)
84{
Benny Halevy738fd0f32011-07-30 20:52:36 -040085 if (nfss->pnfs_curr_ld) {
86 if (nfss->pnfs_curr_ld->clear_layoutdriver)
87 nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
Trond Myklebust2a4c8992012-06-14 13:08:38 -040088 /* Decrement the MDS count. Purge the deviceid cache if zero */
89 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
90 nfs4_deviceid_purge_client(nfss->nfs_client);
Fred Isaman02c35fc2010-10-20 00:17:59 -040091 module_put(nfss->pnfs_curr_ld->owner);
Benny Halevy738fd0f32011-07-30 20:52:36 -040092 }
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040093 nfss->pnfs_curr_ld = NULL;
94}
95
96/*
97 * Try to set the server's pnfs module to the pnfs layout type specified by id.
98 * Currently only one pNFS layout driver per filesystem is supported.
99 *
100 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
101 */
102void
Benny Halevy738fd0f32011-07-30 20:52:36 -0400103set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
104 u32 id)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400105{
106 struct pnfs_layoutdriver_type *ld_type = NULL;
107
108 if (id == 0)
109 goto out_no_driver;
110 if (!(server->nfs_client->cl_exchange_flags &
111 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500112 printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
113 __func__, id, server->nfs_client->cl_exchange_flags);
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400114 goto out_no_driver;
115 }
116 ld_type = find_pnfs_driver(id);
117 if (!ld_type) {
118 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
119 ld_type = find_pnfs_driver(id);
120 if (!ld_type) {
121 dprintk("%s: No pNFS module found for %u.\n",
122 __func__, id);
123 goto out_no_driver;
124 }
125 }
126 server->pnfs_curr_ld = ld_type;
Benny Halevy738fd0f32011-07-30 20:52:36 -0400127 if (ld_type->set_layoutdriver
128 && ld_type->set_layoutdriver(server, mntfh)) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500129 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
130 "driver %u.\n", __func__, id);
Benny Halevy738fd0f32011-07-30 20:52:36 -0400131 module_put(ld_type->owner);
132 goto out_no_driver;
133 }
Trond Myklebust2a4c8992012-06-14 13:08:38 -0400134 /* Bump the MDS count */
135 atomic_inc(&server->nfs_client->cl_mds_count);
Christoph Hellwigea8eecd2011-03-01 01:34:21 +0000136
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400137 dprintk("%s: pNFS module for %u set\n", __func__, id);
138 return;
139
140out_no_driver:
141 dprintk("%s: Using NFSv4 I/O\n", __func__);
142 server->pnfs_curr_ld = NULL;
143}
Fred Isaman02c35fc2010-10-20 00:17:59 -0400144
145int
146pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
147{
148 int status = -EINVAL;
149 struct pnfs_layoutdriver_type *tmp;
150
151 if (ld_type->id == 0) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500152 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
Fred Isaman02c35fc2010-10-20 00:17:59 -0400153 return status;
154 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400155 if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500156 printk(KERN_ERR "NFS: %s Layout driver must provide "
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400157 "alloc_lseg and free_lseg.\n", __func__);
158 return status;
159 }
Fred Isaman02c35fc2010-10-20 00:17:59 -0400160
161 spin_lock(&pnfs_spinlock);
162 tmp = find_pnfs_driver_locked(ld_type->id);
163 if (!tmp) {
164 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
165 status = 0;
166 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
167 ld_type->name);
168 } else {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500169 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
Fred Isaman02c35fc2010-10-20 00:17:59 -0400170 __func__, ld_type->id);
171 }
172 spin_unlock(&pnfs_spinlock);
173
174 return status;
175}
176EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);
177
178void
179pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
180{
181 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
182 spin_lock(&pnfs_spinlock);
183 list_del(&ld_type->pnfs_tblid);
184 spin_unlock(&pnfs_spinlock);
185}
186EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
Benny Halevye5e94012010-10-20 00:18:01 -0400187
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400188/*
189 * pNFS client layout cache
190 */
191
Fred Isamancc6e5342011-01-06 11:36:28 +0000192/* Need to hold i_lock if caller does not already hold reference */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000193void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400194pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
Benny Halevye5e94012010-10-20 00:18:01 -0400195{
Fred Isamancc6e5342011-01-06 11:36:28 +0000196 atomic_inc(&lo->plh_refcount);
197}
198
Benny Halevy636fb9c2011-05-22 19:51:33 +0300199static struct pnfs_layout_hdr *
200pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
201{
202 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
203 return ld->alloc_layout_hdr ? ld->alloc_layout_hdr(ino, gfp_flags) :
204 kzalloc(sizeof(struct pnfs_layout_hdr), gfp_flags);
205}
206
207static void
208pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
209{
210 struct pnfs_layoutdriver_type *ld = NFS_SERVER(lo->plh_inode)->pnfs_curr_ld;
Peng Tao9fa40752011-07-30 20:52:32 -0400211 put_rpccred(lo->plh_lc_cred);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300212 return ld->alloc_layout_hdr ? ld->free_layout_hdr(lo) : kfree(lo);
213}
214
Fred Isamancc6e5342011-01-06 11:36:28 +0000215static void
216destroy_layout_hdr(struct pnfs_layout_hdr *lo)
217{
218 dprintk("%s: freeing layout cache %p\n", __func__, lo);
219 BUG_ON(!list_empty(&lo->plh_layouts));
220 NFS_I(lo->plh_inode)->layout = NULL;
Benny Halevy636fb9c2011-05-22 19:51:33 +0300221 pnfs_free_layout_hdr(lo);
Benny Halevye5e94012010-10-20 00:18:01 -0400222}
223
224static void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400225pnfs_put_layout_hdr_locked(struct pnfs_layout_hdr *lo)
Benny Halevye5e94012010-10-20 00:18:01 -0400226{
Fred Isamancc6e5342011-01-06 11:36:28 +0000227 if (atomic_dec_and_test(&lo->plh_refcount))
228 destroy_layout_hdr(lo);
Benny Halevye5e94012010-10-20 00:18:01 -0400229}
230
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400231void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400232pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
Andy Adamson974cec82010-10-20 00:18:02 -0400233{
Fred Isamancc6e5342011-01-06 11:36:28 +0000234 struct inode *inode = lo->plh_inode;
235
236 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
237 destroy_layout_hdr(lo);
238 spin_unlock(&inode->i_lock);
239 }
Andy Adamson974cec82010-10-20 00:18:02 -0400240}
241
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400242static int
243pnfs_iomode_to_fail_bit(u32 iomode)
244{
245 return iomode == IOMODE_RW ?
246 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
247}
248
249static void
Trond Myklebust3e621212012-09-24 13:07:16 -0400250pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400251{
Trond Myklebust25c75332012-09-18 17:01:12 -0400252 lo->plh_retry_timestamp = jiffies;
Trond Myklebust3e621212012-09-24 13:07:16 -0400253 if (test_and_set_bit(fail_bit, &lo->plh_flags))
254 atomic_inc(&lo->plh_refcount);
255}
256
257static void
258pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
259{
260 if (test_and_clear_bit(fail_bit, &lo->plh_flags))
261 atomic_dec(&lo->plh_refcount);
262}
263
264static void
265pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
266{
267 struct inode *inode = lo->plh_inode;
Trond Myklebust115ce572012-09-20 21:19:43 -0400268 struct pnfs_layout_range range = {
269 .iomode = iomode,
270 .offset = 0,
271 .length = NFS4_MAX_UINT64,
272 };
273 LIST_HEAD(head);
Trond Myklebust3e621212012-09-24 13:07:16 -0400274
275 spin_lock(&inode->i_lock);
276 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
Trond Myklebust115ce572012-09-20 21:19:43 -0400277 pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
Trond Myklebust3e621212012-09-24 13:07:16 -0400278 spin_unlock(&inode->i_lock);
Trond Myklebust115ce572012-09-20 21:19:43 -0400279 pnfs_free_lseg_list(&head);
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400280 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
281 iomode == IOMODE_RW ? "RW" : "READ");
282}
283
284static bool
285pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
286{
Trond Myklebust25c75332012-09-18 17:01:12 -0400287 unsigned long start, end;
Trond Myklebust3e621212012-09-24 13:07:16 -0400288 int fail_bit = pnfs_iomode_to_fail_bit(iomode);
289
290 if (test_bit(fail_bit, &lo->plh_flags) == 0)
Trond Myklebust25c75332012-09-18 17:01:12 -0400291 return false;
292 end = jiffies;
293 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
294 if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
295 /* It is time to retry the failed layoutgets */
Trond Myklebust3e621212012-09-24 13:07:16 -0400296 pnfs_layout_clear_fail_bit(lo, fail_bit);
Trond Myklebust25c75332012-09-18 17:01:12 -0400297 return false;
298 }
299 return true;
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400300}
301
Andy Adamson974cec82010-10-20 00:18:02 -0400302static void
303init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
304{
Fred Isaman566052c2011-01-06 11:36:20 +0000305 INIT_LIST_HEAD(&lseg->pls_list);
Peng Taoa9bae562011-07-30 20:52:33 -0400306 INIT_LIST_HEAD(&lseg->pls_lc_list);
Fred Isaman4541d162011-01-06 11:36:23 +0000307 atomic_set(&lseg->pls_refcount, 1);
308 smp_mb();
309 set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
Fred Isaman566052c2011-01-06 11:36:20 +0000310 lseg->pls_layout = lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400311}
312
Fred Isaman4541d162011-01-06 11:36:23 +0000313static void free_lseg(struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400314{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000315 struct inode *ino = lseg->pls_layout->plh_inode;
Andy Adamson974cec82010-10-20 00:18:02 -0400316
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400317 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400318 /* Matched by pnfs_get_layout_hdr in pnfs_insert_layout */
319 pnfs_put_layout_hdr(NFS_I(ino)->layout);
Andy Adamson974cec82010-10-20 00:18:02 -0400320}
321
Fred Isamand684d2a2011-03-01 01:34:13 +0000322static void
Trond Myklebust9369a432012-09-18 20:57:08 -0400323pnfs_put_lseg_common(struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400324{
Fred Isamand684d2a2011-03-01 01:34:13 +0000325 struct inode *inode = lseg->pls_layout->plh_inode;
326
Benny Halevyd20581a2011-05-22 19:52:03 +0300327 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Fred Isamand684d2a2011-03-01 01:34:13 +0000328 list_del_init(&lseg->pls_list);
329 if (list_empty(&lseg->pls_layout->plh_segs)) {
330 set_bit(NFS_LAYOUT_DESTROYED, &lseg->pls_layout->plh_flags);
331 /* Matched by initial refcount set in alloc_init_layout_hdr */
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400332 pnfs_put_layout_hdr_locked(lseg->pls_layout);
Fred Isamand684d2a2011-03-01 01:34:13 +0000333 }
334 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
335}
336
Fred Isamanbae724e2011-03-01 01:34:15 +0000337void
Trond Myklebust9369a432012-09-18 20:57:08 -0400338pnfs_put_lseg(struct pnfs_layout_segment *lseg)
Fred Isamand684d2a2011-03-01 01:34:13 +0000339{
340 struct inode *inode;
341
342 if (!lseg)
343 return;
344
Fred Isaman4541d162011-01-06 11:36:23 +0000345 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
346 atomic_read(&lseg->pls_refcount),
347 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Fred Isamand684d2a2011-03-01 01:34:13 +0000348 inode = lseg->pls_layout->plh_inode;
349 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
350 LIST_HEAD(free_me);
Andy Adamson974cec82010-10-20 00:18:02 -0400351
Trond Myklebust9369a432012-09-18 20:57:08 -0400352 pnfs_put_lseg_common(lseg);
Fred Isamand684d2a2011-03-01 01:34:13 +0000353 list_add(&lseg->pls_list, &free_me);
354 spin_unlock(&inode->i_lock);
355 pnfs_free_lseg_list(&free_me);
Fred Isaman4541d162011-01-06 11:36:23 +0000356 }
Andy Adamson974cec82010-10-20 00:18:02 -0400357}
Trond Myklebust9369a432012-09-18 20:57:08 -0400358EXPORT_SYMBOL_GPL(pnfs_put_lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400359
Benny Halevyfb3296e2011-05-22 19:47:26 +0300360static inline u64
361end_offset(u64 start, u64 len)
Fred Isaman4541d162011-01-06 11:36:23 +0000362{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300363 u64 end;
364
365 end = start + len;
366 return end >= start ? end : NFS4_MAX_UINT64;
367}
368
369/* last octet in a range */
370static inline u64
371last_byte_offset(u64 start, u64 len)
372{
373 u64 end;
374
375 BUG_ON(!len);
376 end = start + len;
377 return end > start ? end - 1 : NFS4_MAX_UINT64;
378}
379
380/*
381 * is l2 fully contained in l1?
382 * start1 end1
383 * [----------------------------------)
384 * start2 end2
385 * [----------------)
386 */
387static inline int
388lo_seg_contained(struct pnfs_layout_range *l1,
389 struct pnfs_layout_range *l2)
390{
391 u64 start1 = l1->offset;
392 u64 end1 = end_offset(start1, l1->length);
393 u64 start2 = l2->offset;
394 u64 end2 = end_offset(start2, l2->length);
395
396 return (start1 <= start2) && (end1 >= end2);
397}
398
399/*
400 * is l1 and l2 intersecting?
401 * start1 end1
402 * [----------------------------------)
403 * start2 end2
404 * [----------------)
405 */
406static inline int
407lo_seg_intersecting(struct pnfs_layout_range *l1,
408 struct pnfs_layout_range *l2)
409{
410 u64 start1 = l1->offset;
411 u64 end1 = end_offset(start1, l1->length);
412 u64 start2 = l2->offset;
413 u64 end2 = end_offset(start2, l2->length);
414
415 return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
416 (end2 == NFS4_MAX_UINT64 || end2 > start1);
417}
418
Fred Isaman4541d162011-01-06 11:36:23 +0000419static bool
Benny Halevy778b5502011-05-22 19:48:02 +0300420should_free_lseg(struct pnfs_layout_range *lseg_range,
421 struct pnfs_layout_range *recall_range)
Fred Isaman4541d162011-01-06 11:36:23 +0000422{
Benny Halevy778b5502011-05-22 19:48:02 +0300423 return (recall_range->iomode == IOMODE_ANY ||
424 lseg_range->iomode == recall_range->iomode) &&
425 lo_seg_intersecting(lseg_range, recall_range);
Fred Isaman4541d162011-01-06 11:36:23 +0000426}
427
428/* Returns 1 if lseg is removed from list, 0 otherwise */
429static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
430 struct list_head *tmp_list)
431{
432 int rv = 0;
433
434 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
435 /* Remove the reference keeping the lseg in the
436 * list. It will now be removed when all
437 * outstanding io is finished.
438 */
Fred Isamand684d2a2011-03-01 01:34:13 +0000439 dprintk("%s: lseg %p ref %d\n", __func__, lseg,
440 atomic_read(&lseg->pls_refcount));
441 if (atomic_dec_and_test(&lseg->pls_refcount)) {
Trond Myklebust9369a432012-09-18 20:57:08 -0400442 pnfs_put_lseg_common(lseg);
Fred Isamand684d2a2011-03-01 01:34:13 +0000443 list_add(&lseg->pls_list, tmp_list);
444 rv = 1;
445 }
Fred Isaman4541d162011-01-06 11:36:23 +0000446 }
447 return rv;
448}
449
450/* Returns count of number of matching invalid lsegs remaining in list
451 * after call.
452 */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000453int
Trond Myklebust49a85062012-09-18 20:43:31 -0400454pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
Fred Isaman4541d162011-01-06 11:36:23 +0000455 struct list_head *tmp_list,
Benny Halevy778b5502011-05-22 19:48:02 +0300456 struct pnfs_layout_range *recall_range)
Andy Adamson974cec82010-10-20 00:18:02 -0400457{
458 struct pnfs_layout_segment *lseg, *next;
Fred Isaman4541d162011-01-06 11:36:23 +0000459 int invalid = 0, removed = 0;
Andy Adamson974cec82010-10-20 00:18:02 -0400460
461 dprintk("%s:Begin lo %p\n", __func__, lo);
462
Fred Isaman38511722011-02-03 18:28:50 +0000463 if (list_empty(&lo->plh_segs)) {
Andy Adamson2701d082012-05-24 13:13:24 -0400464 /* Reset MDS Threshold I/O counters */
465 NFS_I(lo->plh_inode)->write_io = 0;
466 NFS_I(lo->plh_inode)->read_io = 0;
Fred Isaman38511722011-02-03 18:28:50 +0000467 if (!test_and_set_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags))
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400468 pnfs_put_layout_hdr_locked(lo);
Fred Isaman38511722011-02-03 18:28:50 +0000469 return 0;
470 }
Fred Isaman4541d162011-01-06 11:36:23 +0000471 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
Benny Halevy778b5502011-05-22 19:48:02 +0300472 if (!recall_range ||
473 should_free_lseg(&lseg->pls_range, recall_range)) {
Fred Isaman4541d162011-01-06 11:36:23 +0000474 dprintk("%s: freeing lseg %p iomode %d "
475 "offset %llu length %llu\n", __func__,
476 lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
477 lseg->pls_range.length);
478 invalid++;
479 removed += mark_lseg_invalid(lseg, tmp_list);
480 }
481 dprintk("%s:Return %i\n", __func__, invalid - removed);
482 return invalid - removed;
Andy Adamson974cec82010-10-20 00:18:02 -0400483}
484
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000485/* note free_me must contain lsegs from a single layout_hdr */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000486void
Fred Isaman4541d162011-01-06 11:36:23 +0000487pnfs_free_lseg_list(struct list_head *free_me)
Andy Adamson974cec82010-10-20 00:18:02 -0400488{
Fred Isaman4541d162011-01-06 11:36:23 +0000489 struct pnfs_layout_segment *lseg, *tmp;
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000490 struct pnfs_layout_hdr *lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400491
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000492 if (list_empty(free_me))
493 return;
494
495 lo = list_first_entry(free_me, struct pnfs_layout_segment,
496 pls_list)->pls_layout;
497
498 if (test_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags)) {
499 struct nfs_client *clp;
500
501 clp = NFS_SERVER(lo->plh_inode)->nfs_client;
502 spin_lock(&clp->cl_lock);
503 list_del_init(&lo->plh_layouts);
504 spin_unlock(&clp->cl_lock);
505 }
Fred Isaman4541d162011-01-06 11:36:23 +0000506 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
Fred Isaman566052c2011-01-06 11:36:20 +0000507 list_del(&lseg->pls_list);
Fred Isaman4541d162011-01-06 11:36:23 +0000508 free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400509 }
510}
511
Benny Halevye5e94012010-10-20 00:18:01 -0400512void
513pnfs_destroy_layout(struct nfs_inode *nfsi)
514{
515 struct pnfs_layout_hdr *lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400516 LIST_HEAD(tmp_list);
Benny Halevye5e94012010-10-20 00:18:01 -0400517
518 spin_lock(&nfsi->vfs_inode.i_lock);
519 lo = nfsi->layout;
520 if (lo) {
Fred Isaman38511722011-02-03 18:28:50 +0000521 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
Trond Myklebust49a85062012-09-18 20:43:31 -0400522 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Trond Myklebust3e621212012-09-24 13:07:16 -0400523 pnfs_get_layout_hdr(lo);
524 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
525 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
526 spin_unlock(&nfsi->vfs_inode.i_lock);
527 pnfs_free_lseg_list(&tmp_list);
528 pnfs_put_layout_hdr(lo);
529 } else
530 spin_unlock(&nfsi->vfs_inode.i_lock);
Benny Halevye5e94012010-10-20 00:18:01 -0400531}
Andy Adamson041245c2012-04-27 17:53:53 -0400532EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
Benny Halevye5e94012010-10-20 00:18:01 -0400533
Andy Adamson974cec82010-10-20 00:18:02 -0400534/*
535 * Called by the state manger to remove all layouts established under an
536 * expired lease.
537 */
538void
539pnfs_destroy_all_layouts(struct nfs_client *clp)
540{
Weston Andros Adamson6382a442011-06-01 16:44:44 -0400541 struct nfs_server *server;
Andy Adamson974cec82010-10-20 00:18:02 -0400542 struct pnfs_layout_hdr *lo;
543 LIST_HEAD(tmp_list);
544
Andy Adamsonc47abcf2011-06-15 17:52:40 -0400545 nfs4_deviceid_mark_client_invalid(clp);
546 nfs4_deviceid_purge_client(clp);
547
Andy Adamson974cec82010-10-20 00:18:02 -0400548 spin_lock(&clp->cl_lock);
Weston Andros Adamson6382a442011-06-01 16:44:44 -0400549 rcu_read_lock();
550 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
551 if (!list_empty(&server->layouts))
552 list_splice_init(&server->layouts, &tmp_list);
553 }
554 rcu_read_unlock();
Andy Adamson974cec82010-10-20 00:18:02 -0400555 spin_unlock(&clp->cl_lock);
556
557 while (!list_empty(&tmp_list)) {
558 lo = list_entry(tmp_list.next, struct pnfs_layout_hdr,
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000559 plh_layouts);
Andy Adamson974cec82010-10-20 00:18:02 -0400560 dprintk("%s freeing layout for inode %lu\n", __func__,
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000561 lo->plh_inode->i_ino);
Andy Adamson2887fe42011-05-11 01:19:58 -0400562 list_del_init(&lo->plh_layouts);
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000563 pnfs_destroy_layout(NFS_I(lo->plh_inode));
Andy Adamson974cec82010-10-20 00:18:02 -0400564 }
565}
566
Fred Isamanfd6002e2011-01-06 11:36:22 +0000567/* update lo->plh_stateid with new if is more recent */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000568void
569pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
570 bool update_barrier)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400571{
Fred Isamanfd6002e2011-01-06 11:36:22 +0000572 u32 oldseq, newseq;
Andy Adamson974cec82010-10-20 00:18:02 -0400573
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500574 oldseq = be32_to_cpu(lo->plh_stateid.seqid);
575 newseq = be32_to_cpu(new->seqid);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000576 if ((int)(newseq - oldseq) > 0) {
Trond Myklebustf597c532012-03-04 18:13:56 -0500577 nfs4_stateid_copy(&lo->plh_stateid, new);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000578 if (update_barrier) {
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500579 u32 new_barrier = be32_to_cpu(new->seqid);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000580
581 if ((int)(new_barrier - lo->plh_barrier))
582 lo->plh_barrier = new_barrier;
583 } else {
584 /* Because of wraparound, we want to keep the barrier
585 * "close" to the current seqids. It needs to be
586 * within 2**31 to count as "behind", so if it
587 * gets too near that limit, give us a litle leeway
588 * and bring it to within 2**30.
589 * NOTE - and yes, this is all unsigned arithmetic.
590 */
591 if (unlikely((newseq - lo->plh_barrier) > (3 << 29)))
592 lo->plh_barrier = newseq - (1 << 30);
593 }
594 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400595}
596
Fred Isamancf7d63f2011-01-06 11:36:25 +0000597/* lget is set to 1 if called from inside send_layoutget call chain */
598static bool
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000599pnfs_layoutgets_blocked(struct pnfs_layout_hdr *lo, nfs4_stateid *stateid,
600 int lget)
Fred Isamancf7d63f2011-01-06 11:36:25 +0000601{
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000602 if ((stateid) &&
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500603 (int)(lo->plh_barrier - be32_to_cpu(stateid->seqid)) >= 0)
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000604 return true;
Fred Isamanf7e89172011-01-06 11:36:32 +0000605 return lo->plh_block_lgets ||
Fred Isaman38511722011-02-03 18:28:50 +0000606 test_bit(NFS_LAYOUT_DESTROYED, &lo->plh_flags) ||
Fred Isamanf7e89172011-01-06 11:36:32 +0000607 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000608 (list_empty(&lo->plh_segs) &&
Fred Isamancf7d63f2011-01-06 11:36:25 +0000609 (atomic_read(&lo->plh_outstanding) > lget));
610}
611
Fred Isamanfd6002e2011-01-06 11:36:22 +0000612int
613pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
614 struct nfs4_state *open_state)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400615{
Fred Isamanfd6002e2011-01-06 11:36:22 +0000616 int status = 0;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400617
618 dprintk("--> %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000619 spin_lock(&lo->plh_inode->i_lock);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000620 if (pnfs_layoutgets_blocked(lo, NULL, 1)) {
Fred Isamancf7d63f2011-01-06 11:36:25 +0000621 status = -EAGAIN;
622 } else if (list_empty(&lo->plh_segs)) {
Fred Isamanfd6002e2011-01-06 11:36:22 +0000623 int seq;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400624
Fred Isamanfd6002e2011-01-06 11:36:22 +0000625 do {
626 seq = read_seqbegin(&open_state->seqlock);
Trond Myklebustf597c532012-03-04 18:13:56 -0500627 nfs4_stateid_copy(dst, &open_state->stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000628 } while (read_seqretry(&open_state->seqlock, seq));
629 } else
Trond Myklebustf597c532012-03-04 18:13:56 -0500630 nfs4_stateid_copy(dst, &lo->plh_stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000631 spin_unlock(&lo->plh_inode->i_lock);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400632 dprintk("<-- %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000633 return status;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400634}
635
636/*
637* Get layout from server.
638* for now, assume that whole file layouts are requested.
639* arg->offset: 0
640* arg->length: all ones
641*/
Benny Halevye5e94012010-10-20 00:18:01 -0400642static struct pnfs_layout_segment *
643send_layoutget(struct pnfs_layout_hdr *lo,
644 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +0300645 struct pnfs_layout_range *range,
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400646 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400647{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000648 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400649 struct nfs_server *server = NFS_SERVER(ino);
650 struct nfs4_layoutget *lgp;
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400651 struct pnfs_layout_segment *lseg;
Benny Halevye5e94012010-10-20 00:18:01 -0400652
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400653 dprintk("--> %s\n", __func__);
654
655 BUG_ON(ctx == NULL);
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400656 lgp = kzalloc(sizeof(*lgp), gfp_flags);
Fred Isamancf7d63f2011-01-06 11:36:25 +0000657 if (lgp == NULL)
Andy Adamson974cec82010-10-20 00:18:02 -0400658 return NULL;
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400659
Benny Halevyfb3296e2011-05-22 19:47:26 +0300660 lgp->args.minlength = PAGE_CACHE_SIZE;
661 if (lgp->args.minlength > range->length)
662 lgp->args.minlength = range->length;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400663 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
Benny Halevyfb3296e2011-05-22 19:47:26 +0300664 lgp->args.range = *range;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400665 lgp->args.type = server->pnfs_curr_ld->id;
666 lgp->args.inode = ino;
667 lgp->args.ctx = get_nfs_open_context(ctx);
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400668 lgp->gfp_flags = gfp_flags;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400669
670 /* Synchronously retrieve layout information from server and
671 * store in lseg.
672 */
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400673 lseg = nfs4_proc_layoutget(lgp, gfp_flags);
674 if (IS_ERR(lseg)) {
675 switch (PTR_ERR(lseg)) {
676 case -ENOMEM:
677 case -ERESTARTSYS:
678 break;
679 default:
680 /* remember that LAYOUTGET failed and suspend trying */
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400681 pnfs_layout_io_set_failed(lo, range->iomode);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400682 }
683 return NULL;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400684 }
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400685
Andy Adamson974cec82010-10-20 00:18:02 -0400686 return lseg;
687}
688
Andy Adamson293b3b02012-06-20 15:03:34 -0400689/*
690 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
691 * when the layout segment list is empty.
692 *
693 * Note that a pnfs_layout_hdr can exist with an empty layout segment
694 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
695 * deviceid is marked invalid.
696 */
Benny Halevycbe82602011-05-22 19:52:37 +0300697int
698_pnfs_return_layout(struct inode *ino)
699{
700 struct pnfs_layout_hdr *lo = NULL;
701 struct nfs_inode *nfsi = NFS_I(ino);
702 LIST_HEAD(tmp_list);
703 struct nfs4_layoutreturn *lrp;
704 nfs4_stateid stateid;
Andy Adamson293b3b02012-06-20 15:03:34 -0400705 int status = 0, empty;
Benny Halevycbe82602011-05-22 19:52:37 +0300706
Andy Adamson366d5052012-06-20 15:03:33 -0400707 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
Benny Halevycbe82602011-05-22 19:52:37 +0300708
709 spin_lock(&ino->i_lock);
710 lo = nfsi->layout;
Andy Adamson366d5052012-06-20 15:03:33 -0400711 if (!lo || pnfs_test_layout_returned(lo)) {
Benny Halevycbe82602011-05-22 19:52:37 +0300712 spin_unlock(&ino->i_lock);
Andy Adamson293b3b02012-06-20 15:03:34 -0400713 dprintk("NFS: %s no layout to return\n", __func__);
714 goto out;
Benny Halevycbe82602011-05-22 19:52:37 +0300715 }
716 stateid = nfsi->layout->plh_stateid;
717 /* Reference matched in nfs4_layoutreturn_release */
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400718 pnfs_get_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -0400719 empty = list_empty(&lo->plh_segs);
Trond Myklebust49a85062012-09-18 20:43:31 -0400720 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Andy Adamson293b3b02012-06-20 15:03:34 -0400721 /* Don't send a LAYOUTRETURN if list was initially empty */
722 if (empty) {
723 spin_unlock(&ino->i_lock);
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400724 pnfs_put_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -0400725 dprintk("NFS: %s no layout segments to return\n", __func__);
726 goto out;
727 }
Fred Isamanea0ded72011-06-15 12:31:02 -0400728 lo->plh_block_lgets++;
Andy Adamson366d5052012-06-20 15:03:33 -0400729 pnfs_mark_layout_returned(lo);
Benny Halevycbe82602011-05-22 19:52:37 +0300730 spin_unlock(&ino->i_lock);
731 pnfs_free_lseg_list(&tmp_list);
732
733 WARN_ON(test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags));
734
735 lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
736 if (unlikely(lrp == NULL)) {
737 status = -ENOMEM;
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400738 pnfs_layout_io_set_failed(lo, IOMODE_RW);
739 pnfs_layout_io_set_failed(lo, IOMODE_READ);
Andy Adamson366d5052012-06-20 15:03:33 -0400740 pnfs_clear_layout_returned(lo);
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400741 pnfs_put_layout_hdr(lo);
Benny Halevycbe82602011-05-22 19:52:37 +0300742 goto out;
743 }
744
745 lrp->args.stateid = stateid;
746 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
747 lrp->args.inode = ino;
Trond Myklebusta56aaa02011-06-15 11:59:10 -0400748 lrp->args.layout = lo;
Benny Halevycbe82602011-05-22 19:52:37 +0300749 lrp->clp = NFS_SERVER(ino)->nfs_client;
750
751 status = nfs4_proc_layoutreturn(lrp);
752out:
753 dprintk("<-- %s status: %d\n", __func__, status);
754 return status;
755}
Andy Adamson0a57cda2012-04-27 17:53:50 -0400756EXPORT_SYMBOL_GPL(_pnfs_return_layout);
Benny Halevycbe82602011-05-22 19:52:37 +0300757
Fred Isamanf7e89172011-01-06 11:36:32 +0000758bool pnfs_roc(struct inode *ino)
759{
760 struct pnfs_layout_hdr *lo;
761 struct pnfs_layout_segment *lseg, *tmp;
762 LIST_HEAD(tmp_list);
763 bool found = false;
764
765 spin_lock(&ino->i_lock);
766 lo = NFS_I(ino)->layout;
767 if (!lo || !test_and_clear_bit(NFS_LAYOUT_ROC, &lo->plh_flags) ||
768 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
769 goto out_nolayout;
770 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
771 if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
772 mark_lseg_invalid(lseg, &tmp_list);
773 found = true;
774 }
775 if (!found)
776 goto out_nolayout;
777 lo->plh_block_lgets++;
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400778 pnfs_get_layout_hdr(lo); /* matched in pnfs_roc_release */
Fred Isamanf7e89172011-01-06 11:36:32 +0000779 spin_unlock(&ino->i_lock);
780 pnfs_free_lseg_list(&tmp_list);
781 return true;
782
783out_nolayout:
784 spin_unlock(&ino->i_lock);
785 return false;
786}
787
788void pnfs_roc_release(struct inode *ino)
789{
790 struct pnfs_layout_hdr *lo;
791
792 spin_lock(&ino->i_lock);
793 lo = NFS_I(ino)->layout;
794 lo->plh_block_lgets--;
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400795 pnfs_put_layout_hdr_locked(lo);
Fred Isamanf7e89172011-01-06 11:36:32 +0000796 spin_unlock(&ino->i_lock);
797}
798
799void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
800{
801 struct pnfs_layout_hdr *lo;
802
803 spin_lock(&ino->i_lock);
804 lo = NFS_I(ino)->layout;
805 if ((int)(barrier - lo->plh_barrier) > 0)
806 lo->plh_barrier = barrier;
807 spin_unlock(&ino->i_lock);
808}
809
810bool pnfs_roc_drain(struct inode *ino, u32 *barrier)
811{
812 struct nfs_inode *nfsi = NFS_I(ino);
813 struct pnfs_layout_segment *lseg;
814 bool found = false;
815
816 spin_lock(&ino->i_lock);
817 list_for_each_entry(lseg, &nfsi->layout->plh_segs, pls_list)
818 if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
819 found = true;
820 break;
821 }
822 if (!found) {
823 struct pnfs_layout_hdr *lo = nfsi->layout;
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500824 u32 current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
Fred Isamanf7e89172011-01-06 11:36:32 +0000825
826 /* Since close does not return a layout stateid for use as
827 * a barrier, we choose the worst-case barrier.
828 */
829 *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
830 }
831 spin_unlock(&ino->i_lock);
832 return found;
833}
834
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400835/*
836 * Compare two layout segments for sorting into layout cache.
837 * We want to preferentially return RW over RO layouts, so ensure those
838 * are seen first.
839 */
840static s64
Benny Halevyfb3296e2011-05-22 19:47:26 +0300841cmp_layout(struct pnfs_layout_range *l1,
842 struct pnfs_layout_range *l2)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400843{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300844 s64 d;
845
846 /* high offset > low offset */
847 d = l1->offset - l2->offset;
848 if (d)
849 return d;
850
851 /* short length > long length */
852 d = l2->length - l1->length;
853 if (d)
854 return d;
855
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400856 /* read > read/write */
Benny Halevyfb3296e2011-05-22 19:47:26 +0300857 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400858}
859
Andy Adamson974cec82010-10-20 00:18:02 -0400860static void
861pnfs_insert_layout(struct pnfs_layout_hdr *lo,
862 struct pnfs_layout_segment *lseg)
863{
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400864 struct pnfs_layout_segment *lp;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400865
Andy Adamson974cec82010-10-20 00:18:02 -0400866 dprintk("%s:Begin\n", __func__);
867
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000868 assert_spin_locked(&lo->plh_inode->i_lock);
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000869 list_for_each_entry(lp, &lo->plh_segs, pls_list) {
Benny Halevyfb3296e2011-05-22 19:47:26 +0300870 if (cmp_layout(&lseg->pls_range, &lp->pls_range) > 0)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400871 continue;
Fred Isaman566052c2011-01-06 11:36:20 +0000872 list_add_tail(&lseg->pls_list, &lp->pls_list);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400873 dprintk("%s: inserted lseg %p "
874 "iomode %d offset %llu length %llu before "
875 "lp %p iomode %d offset %llu length %llu\n",
Fred Isaman566052c2011-01-06 11:36:20 +0000876 __func__, lseg, lseg->pls_range.iomode,
877 lseg->pls_range.offset, lseg->pls_range.length,
878 lp, lp->pls_range.iomode, lp->pls_range.offset,
879 lp->pls_range.length);
Benny Halevyfb3296e2011-05-22 19:47:26 +0300880 goto out;
Andy Adamson974cec82010-10-20 00:18:02 -0400881 }
Benny Halevyfb3296e2011-05-22 19:47:26 +0300882 list_add_tail(&lseg->pls_list, &lo->plh_segs);
883 dprintk("%s: inserted lseg %p "
884 "iomode %d offset %llu length %llu at tail\n",
885 __func__, lseg, lseg->pls_range.iomode,
886 lseg->pls_range.offset, lseg->pls_range.length);
887out:
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400888 pnfs_get_layout_hdr(lo);
Andy Adamson974cec82010-10-20 00:18:02 -0400889
890 dprintk("%s:Return\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -0400891}
892
893static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -0400894alloc_init_layout_hdr(struct inode *ino,
895 struct nfs_open_context *ctx,
896 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400897{
898 struct pnfs_layout_hdr *lo;
899
Benny Halevy636fb9c2011-05-22 19:51:33 +0300900 lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -0400901 if (!lo)
902 return NULL;
Fred Isamancc6e5342011-01-06 11:36:28 +0000903 atomic_set(&lo->plh_refcount, 1);
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000904 INIT_LIST_HEAD(&lo->plh_layouts);
905 INIT_LIST_HEAD(&lo->plh_segs);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000906 INIT_LIST_HEAD(&lo->plh_bulk_recall);
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000907 lo->plh_inode = ino;
Peng Tao9fa40752011-07-30 20:52:32 -0400908 lo->plh_lc_cred = get_rpccred(ctx->state->owner->so_cred);
Benny Halevye5e94012010-10-20 00:18:01 -0400909 return lo;
910}
911
912static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -0400913pnfs_find_alloc_layout(struct inode *ino,
914 struct nfs_open_context *ctx,
915 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400916{
917 struct nfs_inode *nfsi = NFS_I(ino);
918 struct pnfs_layout_hdr *new = NULL;
919
920 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);
921
922 assert_spin_locked(&ino->i_lock);
Fred Isaman4541d162011-01-06 11:36:23 +0000923 if (nfsi->layout) {
924 if (test_bit(NFS_LAYOUT_DESTROYED, &nfsi->layout->plh_flags))
925 return NULL;
926 else
927 return nfsi->layout;
928 }
Benny Halevye5e94012010-10-20 00:18:01 -0400929 spin_unlock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -0400930 new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -0400931 spin_lock(&ino->i_lock);
932
933 if (likely(nfsi->layout == NULL)) /* Won the race? */
934 nfsi->layout = new;
935 else
Benny Halevy636fb9c2011-05-22 19:51:33 +0300936 pnfs_free_layout_hdr(new);
Benny Halevye5e94012010-10-20 00:18:01 -0400937 return nfsi->layout;
938}
939
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400940/*
941 * iomode matching rules:
942 * iomode lseg match
943 * ----- ----- -----
944 * ANY READ true
945 * ANY RW true
946 * RW READ false
947 * RW RW true
948 * READ READ true
949 * READ RW true
950 */
951static int
Benny Halevyfb3296e2011-05-22 19:47:26 +0300952is_matching_lseg(struct pnfs_layout_range *ls_range,
953 struct pnfs_layout_range *range)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400954{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300955 struct pnfs_layout_range range1;
956
957 if ((range->iomode == IOMODE_RW &&
958 ls_range->iomode != IOMODE_RW) ||
959 !lo_seg_intersecting(ls_range, range))
960 return 0;
961
962 /* range1 covers only the first byte in the range */
963 range1 = *range;
964 range1.length = 1;
965 return lo_seg_contained(ls_range, &range1);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400966}
967
968/*
969 * lookup range in layout
970 */
Benny Halevye5e94012010-10-20 00:18:01 -0400971static struct pnfs_layout_segment *
Benny Halevyfb3296e2011-05-22 19:47:26 +0300972pnfs_find_lseg(struct pnfs_layout_hdr *lo,
973 struct pnfs_layout_range *range)
Benny Halevye5e94012010-10-20 00:18:01 -0400974{
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400975 struct pnfs_layout_segment *lseg, *ret = NULL;
976
977 dprintk("%s:Begin\n", __func__);
978
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000979 assert_spin_locked(&lo->plh_inode->i_lock);
980 list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
Fred Isaman4541d162011-01-06 11:36:23 +0000981 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
Benny Halevyfb3296e2011-05-22 19:47:26 +0300982 is_matching_lseg(&lseg->pls_range, range)) {
Trond Myklebust9369a432012-09-18 20:57:08 -0400983 ret = pnfs_get_lseg(lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400984 break;
985 }
Benny Halevyd771e3a2011-06-14 16:30:16 -0400986 if (lseg->pls_range.offset > range->offset)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400987 break;
988 }
989
990 dprintk("%s:Return lseg %p ref %d\n",
Fred Isaman4541d162011-01-06 11:36:23 +0000991 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400992 return ret;
Benny Halevye5e94012010-10-20 00:18:01 -0400993}
994
995/*
Andy Adamsond23d61c2012-05-23 05:02:37 -0400996 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
997 * to the MDS or over pNFS
998 *
999 * The nfs_inode read_io and write_io fields are cumulative counters reset
1000 * when there are no layout segments. Note that in pnfs_update_layout iomode
1001 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
1002 * WRITE request.
1003 *
1004 * A return of true means use MDS I/O.
1005 *
1006 * From rfc 5661:
1007 * If a file's size is smaller than the file size threshold, data accesses
1008 * SHOULD be sent to the metadata server. If an I/O request has a length that
1009 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
1010 * server. If both file size and I/O size are provided, the client SHOULD
1011 * reach or exceed both thresholds before sending its read or write
1012 * requests to the data server.
1013 */
1014static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
1015 struct inode *ino, int iomode)
1016{
1017 struct nfs4_threshold *t = ctx->mdsthreshold;
1018 struct nfs_inode *nfsi = NFS_I(ino);
1019 loff_t fsize = i_size_read(ino);
1020 bool size = false, size_set = false, io = false, io_set = false, ret = false;
1021
1022 if (t == NULL)
1023 return ret;
1024
1025 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
1026 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);
1027
1028 switch (iomode) {
1029 case IOMODE_READ:
1030 if (t->bm & THRESHOLD_RD) {
1031 dprintk("%s fsize %llu\n", __func__, fsize);
1032 size_set = true;
1033 if (fsize < t->rd_sz)
1034 size = true;
1035 }
1036 if (t->bm & THRESHOLD_RD_IO) {
1037 dprintk("%s nfsi->read_io %llu\n", __func__,
1038 nfsi->read_io);
1039 io_set = true;
1040 if (nfsi->read_io < t->rd_io_sz)
1041 io = true;
1042 }
1043 break;
1044 case IOMODE_RW:
1045 if (t->bm & THRESHOLD_WR) {
1046 dprintk("%s fsize %llu\n", __func__, fsize);
1047 size_set = true;
1048 if (fsize < t->wr_sz)
1049 size = true;
1050 }
1051 if (t->bm & THRESHOLD_WR_IO) {
1052 dprintk("%s nfsi->write_io %llu\n", __func__,
1053 nfsi->write_io);
1054 io_set = true;
1055 if (nfsi->write_io < t->wr_io_sz)
1056 io = true;
1057 }
1058 break;
1059 }
1060 if (size_set && io_set) {
1061 if (size && io)
1062 ret = true;
1063 } else if (size || io)
1064 ret = true;
1065
1066 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
1067 return ret;
1068}
1069
1070/*
Benny Halevye5e94012010-10-20 00:18:01 -04001071 * Layout segment is retreived from the server if not cached.
1072 * The appropriate layout segment is referenced and returned to the caller.
1073 */
Andy Adamson7c24d942011-06-13 18:22:38 -04001074struct pnfs_layout_segment *
Benny Halevye5e94012010-10-20 00:18:01 -04001075pnfs_update_layout(struct inode *ino,
1076 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +03001077 loff_t pos,
1078 u64 count,
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001079 enum pnfs_iomode iomode,
1080 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001081{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001082 struct pnfs_layout_range arg = {
1083 .iomode = iomode,
1084 .offset = pos,
1085 .length = count,
1086 };
Benny Halevy707ed5f2011-05-22 19:47:46 +03001087 unsigned pg_offset;
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001088 struct nfs_server *server = NFS_SERVER(ino);
1089 struct nfs_client *clp = server->nfs_client;
Benny Halevye5e94012010-10-20 00:18:01 -04001090 struct pnfs_layout_hdr *lo;
1091 struct pnfs_layout_segment *lseg = NULL;
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001092 bool first = false;
Benny Halevye5e94012010-10-20 00:18:01 -04001093
1094 if (!pnfs_enabled_sb(NFS_SERVER(ino)))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001095 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001096
1097 if (pnfs_within_mdsthreshold(ctx, ino, iomode))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001098 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001099
Benny Halevye5e94012010-10-20 00:18:01 -04001100 spin_lock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001101 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001102 if (lo == NULL) {
1103 spin_unlock(&ino->i_lock);
1104 goto out;
1105 }
Benny Halevye5e94012010-10-20 00:18:01 -04001106
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001107 /* Do we even need to bother with this? */
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001108 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001109 dprintk("%s matches recall, use MDS\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001110 goto out_unlock;
1111 }
1112
1113 /* if LAYOUTGET already failed once we don't try again */
Trond Myklebustb9e028f2012-09-18 16:41:18 -04001114 if (pnfs_layout_io_test_failed(lo, iomode))
Benny Halevye5e94012010-10-20 00:18:01 -04001115 goto out_unlock;
1116
Andy Adamson568e8c42011-03-01 01:34:22 +00001117 /* Check to see if the layout for the given range already exists */
Benny Halevyfb3296e2011-05-22 19:47:26 +03001118 lseg = pnfs_find_lseg(lo, &arg);
Andy Adamson568e8c42011-03-01 01:34:22 +00001119 if (lseg)
1120 goto out_unlock;
1121
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001122 if (pnfs_layoutgets_blocked(lo, NULL, 0))
Fred Isamancf7d63f2011-01-06 11:36:25 +00001123 goto out_unlock;
1124 atomic_inc(&lo->plh_outstanding);
1125
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001126 pnfs_get_layout_hdr(lo);
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001127 if (list_empty(&lo->plh_segs))
1128 first = true;
Andy Adamson366d5052012-06-20 15:03:33 -04001129
1130 /* Enable LAYOUTRETURNs */
1131 pnfs_clear_layout_returned(lo);
1132
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001133 spin_unlock(&ino->i_lock);
1134 if (first) {
Fred Isaman2130ff62011-01-06 11:36:26 +00001135 /* The lo must be on the clp list if there is any
1136 * chance of a CB_LAYOUTRECALL(FILE) coming in.
1137 */
1138 spin_lock(&clp->cl_lock);
1139 BUG_ON(!list_empty(&lo->plh_layouts));
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001140 list_add_tail(&lo->plh_layouts, &server->layouts);
Fred Isaman2130ff62011-01-06 11:36:26 +00001141 spin_unlock(&clp->cl_lock);
1142 }
Benny Halevye5e94012010-10-20 00:18:01 -04001143
Benny Halevy707ed5f2011-05-22 19:47:46 +03001144 pg_offset = arg.offset & ~PAGE_CACHE_MASK;
1145 if (pg_offset) {
1146 arg.offset -= pg_offset;
1147 arg.length += pg_offset;
1148 }
Andy Adamson7c24d942011-06-13 18:22:38 -04001149 if (arg.length != NFS4_MAX_UINT64)
1150 arg.length = PAGE_CACHE_ALIGN(arg.length);
Benny Halevy707ed5f2011-05-22 19:47:46 +03001151
Benny Halevyfb3296e2011-05-22 19:47:26 +03001152 lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001153 if (!lseg && first) {
1154 spin_lock(&clp->cl_lock);
1155 list_del_init(&lo->plh_layouts);
1156 spin_unlock(&clp->cl_lock);
Fred Isaman2130ff62011-01-06 11:36:26 +00001157 }
Fred Isamancf7d63f2011-01-06 11:36:25 +00001158 atomic_dec(&lo->plh_outstanding);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001159out_put_layout_hdr:
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001160 pnfs_put_layout_hdr(lo);
Benny Halevye5e94012010-10-20 00:18:01 -04001161out:
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001162 dprintk("%s: inode %s/%llu pNFS layout segment %s for "
1163 "(%s, offset: %llu, length: %llu)\n",
1164 __func__, ino->i_sb->s_id,
1165 (unsigned long long)NFS_FILEID(ino),
1166 lseg == NULL ? "not found" : "found",
1167 iomode==IOMODE_RW ? "read/write" : "read-only",
1168 (unsigned long long)pos,
1169 (unsigned long long)count);
Benny Halevye5e94012010-10-20 00:18:01 -04001170 return lseg;
1171out_unlock:
1172 spin_unlock(&ino->i_lock);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001173 goto out_put_layout_hdr;
Benny Halevye5e94012010-10-20 00:18:01 -04001174}
Andy Adamson7c24d942011-06-13 18:22:38 -04001175EXPORT_SYMBOL_GPL(pnfs_update_layout);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001176
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001177struct pnfs_layout_segment *
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001178pnfs_layout_process(struct nfs4_layoutget *lgp)
1179{
1180 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
1181 struct nfs4_layoutget_res *res = &lgp->res;
1182 struct pnfs_layout_segment *lseg;
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001183 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001184 int status = 0;
1185
1186 /* Inject layout blob into I/O device driver */
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001187 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001188 if (!lseg || IS_ERR(lseg)) {
1189 if (!lseg)
1190 status = -ENOMEM;
1191 else
1192 status = PTR_ERR(lseg);
1193 dprintk("%s: Could not allocate layout: error %d\n",
1194 __func__, status);
1195 goto out;
1196 }
1197
1198 spin_lock(&ino->i_lock);
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001199 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001200 dprintk("%s forget reply due to recall\n", __func__);
1201 goto out_forget_reply;
1202 }
1203
1204 if (pnfs_layoutgets_blocked(lo, &res->stateid, 1)) {
1205 dprintk("%s forget reply due to state\n", __func__);
1206 goto out_forget_reply;
1207 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001208 init_lseg(lo, lseg);
Fred Isaman566052c2011-01-06 11:36:20 +00001209 lseg->pls_range = res->range;
Trond Myklebust9369a432012-09-18 20:57:08 -04001210 pnfs_get_lseg(lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001211 pnfs_insert_layout(lo, lseg);
1212
Fred Isamanf7e89172011-01-06 11:36:32 +00001213 if (res->return_on_close) {
1214 set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
1215 set_bit(NFS_LAYOUT_ROC, &lo->plh_flags);
1216 }
1217
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001218 /* Done processing layoutget. Set the layout stateid */
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001219 pnfs_set_layout_stateid(lo, &res->stateid, false);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001220 spin_unlock(&ino->i_lock);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001221 return lseg;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001222out:
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001223 return ERR_PTR(status);
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001224
1225out_forget_reply:
1226 spin_unlock(&ino->i_lock);
1227 lseg->pls_layout = lo;
1228 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
1229 goto out;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001230}
1231
Trond Myklebustd8007d42011-06-10 13:30:23 -04001232void
1233pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
1234{
1235 BUG_ON(pgio->pg_lseg != NULL);
1236
Fred Isaman1825a0d2012-04-20 19:55:31 -04001237 if (req->wb_offset != req->wb_pgbase) {
1238 nfs_pageio_reset_read_mds(pgio);
1239 return;
1240 }
Trond Myklebustd8007d42011-06-10 13:30:23 -04001241 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1242 req->wb_context,
1243 req_offset(req),
1244 req->wb_bytes,
1245 IOMODE_READ,
1246 GFP_KERNEL);
Trond Myklebuste885de12011-06-10 13:30:23 -04001247 /* If no lseg, fall back to read through mds */
1248 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001249 nfs_pageio_reset_read_mds(pgio);
Trond Myklebuste885de12011-06-10 13:30:23 -04001250
Trond Myklebustd8007d42011-06-10 13:30:23 -04001251}
1252EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);
1253
1254void
1255pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
1256{
1257 BUG_ON(pgio->pg_lseg != NULL);
1258
Fred Isaman1825a0d2012-04-20 19:55:31 -04001259 if (req->wb_offset != req->wb_pgbase) {
1260 nfs_pageio_reset_write_mds(pgio);
1261 return;
1262 }
Trond Myklebustd8007d42011-06-10 13:30:23 -04001263 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1264 req->wb_context,
1265 req_offset(req),
1266 req->wb_bytes,
1267 IOMODE_RW,
1268 GFP_NOFS);
Trond Myklebuste885de12011-06-10 13:30:23 -04001269 /* If no lseg, fall back to write through mds */
1270 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001271 nfs_pageio_reset_write_mds(pgio);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001272}
1273EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);
1274
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001275void
Fred Isaman061ae2e2012-04-20 14:47:48 -04001276pnfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, struct inode *inode,
1277 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust1751c362011-06-10 13:30:23 -04001278{
1279 struct nfs_server *server = NFS_SERVER(inode);
1280 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
1281
1282 if (ld == NULL)
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001283 nfs_pageio_init_read(pgio, inode, compl_ops);
1284 else
1285 nfs_pageio_init(pgio, inode, ld->pg_read_ops, compl_ops, server->rsize, 0);
Trond Myklebust1751c362011-06-10 13:30:23 -04001286}
1287
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001288void
Fred Isaman061ae2e2012-04-20 14:47:48 -04001289pnfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, struct inode *inode,
1290 int ioflags,
1291 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust1751c362011-06-10 13:30:23 -04001292{
1293 struct nfs_server *server = NFS_SERVER(inode);
1294 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
1295
1296 if (ld == NULL)
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001297 nfs_pageio_init_write(pgio, inode, ioflags, compl_ops);
1298 else
1299 nfs_pageio_init(pgio, inode, ld->pg_write_ops, compl_ops, server->wsize, ioflags);
Trond Myklebust1751c362011-06-10 13:30:23 -04001300}
1301
1302bool
Benny Halevydfed2062011-05-25 20:25:22 +03001303pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
1304 struct nfs_page *req)
Fred Isamanbae724e2011-03-01 01:34:15 +00001305{
Trond Myklebustd8007d42011-06-10 13:30:23 -04001306 if (pgio->pg_lseg == NULL)
1307 return nfs_generic_pg_test(pgio, prev, req);
Benny Halevy89a58e32011-05-25 20:54:40 +03001308
Trond Myklebust19982ba2011-06-10 13:30:23 -04001309 /*
1310 * Test if a nfs_page is fully contained in the pnfs_layout_range.
1311 * Note that this test makes several assumptions:
1312 * - that the previous nfs_page in the struct nfs_pageio_descriptor
1313 * is known to lie within the range.
1314 * - that the nfs_page being tested is known to be contiguous with the
1315 * previous nfs_page.
1316 * - Layout ranges are page aligned, so we only have to test the
1317 * start offset of the request.
1318 *
1319 * Please also note that 'end_offset' is actually the offset of the
1320 * first byte that lies outside the pnfs_layout_range. FIXME?
1321 *
1322 */
1323 return req_offset(req) < end_offset(pgio->pg_lseg->pls_range.offset,
1324 pgio->pg_lseg->pls_range.length);
Fred Isamanbae724e2011-03-01 01:34:15 +00001325}
Benny Halevy89a58e32011-05-25 20:54:40 +03001326EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
Fred Isamanbae724e2011-03-01 01:34:15 +00001327
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001328int pnfs_write_done_resend_to_mds(struct inode *inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001329 struct list_head *head,
1330 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001331{
1332 struct nfs_pageio_descriptor pgio;
1333 LIST_HEAD(failed);
1334
1335 /* Resend all requests through the MDS */
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001336 nfs_pageio_init_write(&pgio, inode, FLUSH_STABLE, compl_ops);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001337 while (!list_empty(head)) {
1338 struct nfs_page *req = nfs_list_entry(head->next);
1339
1340 nfs_list_remove_request(req);
1341 if (!nfs_pageio_add_request(&pgio, req))
1342 nfs_list_add_request(req, &failed);
1343 }
1344 nfs_pageio_complete(&pgio);
1345
1346 if (!list_empty(&failed)) {
1347 /* For some reason our attempt to resend pages. Mark the
1348 * overall send request as having failed, and let
1349 * nfs_writeback_release_full deal with the error.
1350 */
1351 list_move(&failed, head);
1352 return -EIO;
1353 }
1354 return 0;
1355}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001356EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001357
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001358static void pnfs_ld_handle_write_error(struct nfs_write_data *data)
1359{
Fred Isamancd841602012-04-20 14:47:44 -04001360 struct nfs_pgio_header *hdr = data->header;
1361
1362 dprintk("pnfs write error = %d\n", hdr->pnfs_error);
1363 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001364 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04001365 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(hdr->inode)->flags);
1366 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001367 }
Fred Isaman6c75dc02012-04-20 14:47:47 -04001368 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1369 data->task.tk_status = pnfs_write_done_resend_to_mds(hdr->inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001370 &hdr->pages,
1371 hdr->completion_ops);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001372}
1373
Benny Halevyd20581a2011-05-22 19:52:03 +03001374/*
1375 * Called by non rpc-based layout drivers
1376 */
Peng Tao8ce160c2011-09-22 21:50:14 -04001377void pnfs_ld_write_done(struct nfs_write_data *data)
Fred Isaman94ad1c82011-03-01 01:34:14 +00001378{
Fred Isamancd841602012-04-20 14:47:44 -04001379 struct nfs_pgio_header *hdr = data->header;
1380
1381 if (!hdr->pnfs_error) {
Benny Halevyd20581a2011-05-22 19:52:03 +03001382 pnfs_set_layoutcommit(data);
Fred Isamancd841602012-04-20 14:47:44 -04001383 hdr->mds_ops->rpc_call_done(&data->task, data);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001384 } else
1385 pnfs_ld_handle_write_error(data);
Fred Isamancd841602012-04-20 14:47:44 -04001386 hdr->mds_ops->rpc_release(data);
Fred Isaman44b83792011-03-03 15:13:44 +00001387}
Benny Halevyd20581a2011-05-22 19:52:03 +03001388EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
Fred Isaman44b83792011-03-03 15:13:44 +00001389
Trond Myklebustdce81292011-07-13 15:59:19 -04001390static void
1391pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
1392 struct nfs_write_data *data)
1393{
Fred Isamancd841602012-04-20 14:47:44 -04001394 struct nfs_pgio_header *hdr = data->header;
1395
Fred Isaman6c75dc02012-04-20 14:47:47 -04001396 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1397 list_splice_tail_init(&hdr->pages, &desc->pg_list);
1398 nfs_pageio_reset_write_mds(desc);
1399 desc->pg_recoalesce = 1;
1400 }
Trond Myklebustdce81292011-07-13 15:59:19 -04001401 nfs_writedata_release(data);
1402}
1403
1404static enum pnfs_try_status
Andy Adamson0382b742011-03-03 15:13:45 +00001405pnfs_try_to_write_data(struct nfs_write_data *wdata,
Trond Myklebustdce81292011-07-13 15:59:19 -04001406 const struct rpc_call_ops *call_ops,
1407 struct pnfs_layout_segment *lseg,
1408 int how)
Andy Adamson0382b742011-03-03 15:13:45 +00001409{
Fred Isamancd841602012-04-20 14:47:44 -04001410 struct nfs_pgio_header *hdr = wdata->header;
1411 struct inode *inode = hdr->inode;
Andy Adamson0382b742011-03-03 15:13:45 +00001412 enum pnfs_try_status trypnfs;
1413 struct nfs_server *nfss = NFS_SERVER(inode);
1414
Fred Isamancd841602012-04-20 14:47:44 -04001415 hdr->mds_ops = call_ops;
Andy Adamson0382b742011-03-03 15:13:45 +00001416
1417 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
1418 inode->i_ino, wdata->args.count, wdata->args.offset, how);
Andy Adamson0382b742011-03-03 15:13:45 +00001419 trypnfs = nfss->pnfs_curr_ld->write_pagelist(wdata, how);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001420 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson0382b742011-03-03 15:13:45 +00001421 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
Andy Adamson0382b742011-03-03 15:13:45 +00001422 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
1423 return trypnfs;
1424}
1425
Trond Myklebustdce81292011-07-13 15:59:19 -04001426static void
1427pnfs_do_multiple_writes(struct nfs_pageio_descriptor *desc, struct list_head *head, int how)
1428{
1429 struct nfs_write_data *data;
1430 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
1431 struct pnfs_layout_segment *lseg = desc->pg_lseg;
1432
1433 desc->pg_lseg = NULL;
1434 while (!list_empty(head)) {
1435 enum pnfs_try_status trypnfs;
1436
Fred Isaman6c75dc02012-04-20 14:47:47 -04001437 data = list_first_entry(head, struct nfs_write_data, list);
Trond Myklebustdce81292011-07-13 15:59:19 -04001438 list_del_init(&data->list);
1439
1440 trypnfs = pnfs_try_to_write_data(data, call_ops, lseg, how);
1441 if (trypnfs == PNFS_NOT_ATTEMPTED)
1442 pnfs_write_through_mds(desc, data);
1443 }
Trond Myklebust9369a432012-09-18 20:57:08 -04001444 pnfs_put_lseg(lseg);
Trond Myklebustdce81292011-07-13 15:59:19 -04001445}
1446
Fred Isaman6c75dc02012-04-20 14:47:47 -04001447static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
1448{
Trond Myklebust9369a432012-09-18 20:57:08 -04001449 pnfs_put_lseg(hdr->lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001450 nfs_writehdr_free(hdr);
1451}
Bryan Schumaker89d77c82012-07-30 16:05:25 -04001452EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001453
Trond Myklebustdce81292011-07-13 15:59:19 -04001454int
1455pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
1456{
Fred Isaman6c75dc02012-04-20 14:47:47 -04001457 struct nfs_write_header *whdr;
1458 struct nfs_pgio_header *hdr;
Trond Myklebustdce81292011-07-13 15:59:19 -04001459 int ret;
1460
Fred Isaman6c75dc02012-04-20 14:47:47 -04001461 whdr = nfs_writehdr_alloc();
1462 if (!whdr) {
Trond Myklebust9b5415b2012-04-27 14:31:47 -04001463 desc->pg_completion_ops->error_cleanup(&desc->pg_list);
Trond Myklebust9369a432012-09-18 20:57:08 -04001464 pnfs_put_lseg(desc->pg_lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001465 desc->pg_lseg = NULL;
1466 return -ENOMEM;
1467 }
1468 hdr = &whdr->header;
1469 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04001470 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001471 atomic_inc(&hdr->refcnt);
1472 ret = nfs_generic_flush(desc, hdr);
Trond Myklebustdce81292011-07-13 15:59:19 -04001473 if (ret != 0) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001474 pnfs_put_lseg(desc->pg_lseg);
Trond Myklebustdce81292011-07-13 15:59:19 -04001475 desc->pg_lseg = NULL;
Fred Isaman6c75dc02012-04-20 14:47:47 -04001476 } else
1477 pnfs_do_multiple_writes(desc, &hdr->rpc_list, desc->pg_ioflags);
1478 if (atomic_dec_and_test(&hdr->refcnt))
Fred Isaman061ae2e2012-04-20 14:47:48 -04001479 hdr->completion_ops->completion(hdr);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001480 return ret;
Trond Myklebustdce81292011-07-13 15:59:19 -04001481}
1482EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
1483
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001484int pnfs_read_done_resend_to_mds(struct inode *inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001485 struct list_head *head,
1486 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust62e4a762011-11-10 14:30:37 -05001487{
1488 struct nfs_pageio_descriptor pgio;
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001489 LIST_HEAD(failed);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001490
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001491 /* Resend all requests through the MDS */
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001492 nfs_pageio_init_read(&pgio, inode, compl_ops);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001493 while (!list_empty(head)) {
1494 struct nfs_page *req = nfs_list_entry(head->next);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001495
1496 nfs_list_remove_request(req);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001497 if (!nfs_pageio_add_request(&pgio, req))
1498 nfs_list_add_request(req, &failed);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001499 }
1500 nfs_pageio_complete(&pgio);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001501
1502 if (!list_empty(&failed)) {
1503 list_move(&failed, head);
1504 return -EIO;
1505 }
1506 return 0;
1507}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001508EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001509
1510static void pnfs_ld_handle_read_error(struct nfs_read_data *data)
1511{
Fred Isamancd841602012-04-20 14:47:44 -04001512 struct nfs_pgio_header *hdr = data->header;
1513
1514 dprintk("pnfs read error = %d\n", hdr->pnfs_error);
1515 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001516 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04001517 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(hdr->inode)->flags);
1518 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001519 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001520 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1521 data->task.tk_status = pnfs_read_done_resend_to_mds(hdr->inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001522 &hdr->pages,
1523 hdr->completion_ops);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001524}
1525
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001526/*
Benny Halevyd20581a2011-05-22 19:52:03 +03001527 * Called by non rpc-based layout drivers
1528 */
Peng Tao9b7eecd2011-09-22 21:50:15 -04001529void pnfs_ld_read_done(struct nfs_read_data *data)
Benny Halevyd20581a2011-05-22 19:52:03 +03001530{
Fred Isamancd841602012-04-20 14:47:44 -04001531 struct nfs_pgio_header *hdr = data->header;
1532
1533 if (likely(!hdr->pnfs_error)) {
Benny Halevyd20581a2011-05-22 19:52:03 +03001534 __nfs4_read_done_cb(data);
Fred Isamancd841602012-04-20 14:47:44 -04001535 hdr->mds_ops->rpc_call_done(&data->task, data);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001536 } else
1537 pnfs_ld_handle_read_error(data);
Fred Isamancd841602012-04-20 14:47:44 -04001538 hdr->mds_ops->rpc_release(data);
Benny Halevyd20581a2011-05-22 19:52:03 +03001539}
1540EXPORT_SYMBOL_GPL(pnfs_ld_read_done);
1541
Trond Myklebust493292d2011-07-13 15:58:28 -04001542static void
1543pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
1544 struct nfs_read_data *data)
1545{
Fred Isamancd841602012-04-20 14:47:44 -04001546 struct nfs_pgio_header *hdr = data->header;
1547
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001548 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1549 list_splice_tail_init(&hdr->pages, &desc->pg_list);
1550 nfs_pageio_reset_read_mds(desc);
1551 desc->pg_recoalesce = 1;
1552 }
Trond Myklebust493292d2011-07-13 15:58:28 -04001553 nfs_readdata_release(data);
1554}
1555
Benny Halevyd20581a2011-05-22 19:52:03 +03001556/*
Andy Adamson64419a92011-03-01 01:34:16 +00001557 * Call the appropriate parallel I/O subsystem read function.
1558 */
Trond Myklebust493292d2011-07-13 15:58:28 -04001559static enum pnfs_try_status
Andy Adamson64419a92011-03-01 01:34:16 +00001560pnfs_try_to_read_data(struct nfs_read_data *rdata,
Trond Myklebust493292d2011-07-13 15:58:28 -04001561 const struct rpc_call_ops *call_ops,
1562 struct pnfs_layout_segment *lseg)
Andy Adamson64419a92011-03-01 01:34:16 +00001563{
Fred Isamancd841602012-04-20 14:47:44 -04001564 struct nfs_pgio_header *hdr = rdata->header;
1565 struct inode *inode = hdr->inode;
Andy Adamson64419a92011-03-01 01:34:16 +00001566 struct nfs_server *nfss = NFS_SERVER(inode);
1567 enum pnfs_try_status trypnfs;
1568
Fred Isamancd841602012-04-20 14:47:44 -04001569 hdr->mds_ops = call_ops;
Andy Adamson64419a92011-03-01 01:34:16 +00001570
1571 dprintk("%s: Reading ino:%lu %u@%llu\n",
1572 __func__, inode->i_ino, rdata->args.count, rdata->args.offset);
1573
1574 trypnfs = nfss->pnfs_curr_ld->read_pagelist(rdata);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001575 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson64419a92011-03-01 01:34:16 +00001576 nfs_inc_stats(inode, NFSIOS_PNFS_READ);
Andy Adamson64419a92011-03-01 01:34:16 +00001577 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
1578 return trypnfs;
1579}
Andy Adamson863a3c62011-03-23 13:27:54 +00001580
Trond Myklebust493292d2011-07-13 15:58:28 -04001581static void
1582pnfs_do_multiple_reads(struct nfs_pageio_descriptor *desc, struct list_head *head)
1583{
1584 struct nfs_read_data *data;
1585 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
1586 struct pnfs_layout_segment *lseg = desc->pg_lseg;
1587
1588 desc->pg_lseg = NULL;
1589 while (!list_empty(head)) {
1590 enum pnfs_try_status trypnfs;
1591
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001592 data = list_first_entry(head, struct nfs_read_data, list);
Trond Myklebust493292d2011-07-13 15:58:28 -04001593 list_del_init(&data->list);
1594
1595 trypnfs = pnfs_try_to_read_data(data, call_ops, lseg);
1596 if (trypnfs == PNFS_NOT_ATTEMPTED)
1597 pnfs_read_through_mds(desc, data);
1598 }
Trond Myklebust9369a432012-09-18 20:57:08 -04001599 pnfs_put_lseg(lseg);
Trond Myklebust493292d2011-07-13 15:58:28 -04001600}
1601
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001602static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
1603{
Trond Myklebust9369a432012-09-18 20:57:08 -04001604 pnfs_put_lseg(hdr->lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001605 nfs_readhdr_free(hdr);
1606}
Bryan Schumaker89d77c82012-07-30 16:05:25 -04001607EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001608
Trond Myklebust493292d2011-07-13 15:58:28 -04001609int
1610pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
1611{
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001612 struct nfs_read_header *rhdr;
1613 struct nfs_pgio_header *hdr;
Trond Myklebust493292d2011-07-13 15:58:28 -04001614 int ret;
1615
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001616 rhdr = nfs_readhdr_alloc();
1617 if (!rhdr) {
Fred Isaman061ae2e2012-04-20 14:47:48 -04001618 desc->pg_completion_ops->error_cleanup(&desc->pg_list);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001619 ret = -ENOMEM;
Trond Myklebust9369a432012-09-18 20:57:08 -04001620 pnfs_put_lseg(desc->pg_lseg);
Trond Myklebust493292d2011-07-13 15:58:28 -04001621 desc->pg_lseg = NULL;
1622 return ret;
1623 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001624 hdr = &rhdr->header;
1625 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04001626 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001627 atomic_inc(&hdr->refcnt);
1628 ret = nfs_generic_pagein(desc, hdr);
1629 if (ret != 0) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001630 pnfs_put_lseg(desc->pg_lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001631 desc->pg_lseg = NULL;
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001632 } else
1633 pnfs_do_multiple_reads(desc, &hdr->rpc_list);
1634 if (atomic_dec_and_test(&hdr->refcnt))
Fred Isaman061ae2e2012-04-20 14:47:48 -04001635 hdr->completion_ops->completion(hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001636 return ret;
Trond Myklebust493292d2011-07-13 15:58:28 -04001637}
1638EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);
1639
Andy Adamson863a3c62011-03-23 13:27:54 +00001640/*
Peng Taoa9bae562011-07-30 20:52:33 -04001641 * There can be multiple RW segments.
Andy Adamson863a3c62011-03-23 13:27:54 +00001642 */
Peng Taoa9bae562011-07-30 20:52:33 -04001643static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
Andy Adamson863a3c62011-03-23 13:27:54 +00001644{
Peng Taoa9bae562011-07-30 20:52:33 -04001645 struct pnfs_layout_segment *lseg;
Andy Adamson863a3c62011-03-23 13:27:54 +00001646
Peng Taoa9bae562011-07-30 20:52:33 -04001647 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
1648 if (lseg->pls_range.iomode == IOMODE_RW &&
1649 test_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
1650 list_add(&lseg->pls_lc_list, listp);
1651 }
Andy Adamson863a3c62011-03-23 13:27:54 +00001652}
1653
Peng Tao1b0ae062011-09-22 21:50:12 -04001654void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
1655{
Trond Myklebustb9e028f2012-09-18 16:41:18 -04001656 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
Peng Tao1b0ae062011-09-22 21:50:12 -04001657}
1658EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
1659
Andy Adamson863a3c62011-03-23 13:27:54 +00001660void
1661pnfs_set_layoutcommit(struct nfs_write_data *wdata)
1662{
Fred Isamancd841602012-04-20 14:47:44 -04001663 struct nfs_pgio_header *hdr = wdata->header;
1664 struct inode *inode = hdr->inode;
1665 struct nfs_inode *nfsi = NFS_I(inode);
Vitaliy Gusev4b8ee2b2011-05-20 01:34:46 +04001666 loff_t end_pos = wdata->mds_offset + wdata->res.count;
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001667 bool mark_as_dirty = false;
Andy Adamson863a3c62011-03-23 13:27:54 +00001668
Fred Isamancd841602012-04-20 14:47:44 -04001669 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00001670 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001671 mark_as_dirty = true;
Andy Adamson863a3c62011-03-23 13:27:54 +00001672 dprintk("%s: Set layoutcommit for inode %lu ",
Fred Isamancd841602012-04-20 14:47:44 -04001673 __func__, inode->i_ino);
Andy Adamson863a3c62011-03-23 13:27:54 +00001674 }
Fred Isamancd841602012-04-20 14:47:44 -04001675 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &hdr->lseg->pls_flags)) {
Peng Taoa9bae562011-07-30 20:52:33 -04001676 /* references matched in nfs4_layoutcommit_release */
Trond Myklebust9369a432012-09-18 20:57:08 -04001677 pnfs_get_lseg(hdr->lseg);
Peng Taoa9bae562011-07-30 20:52:33 -04001678 }
Peng Taoacff58802011-07-30 20:52:31 -04001679 if (end_pos > nfsi->layout->plh_lwb)
1680 nfsi->layout->plh_lwb = end_pos;
Fred Isamancd841602012-04-20 14:47:44 -04001681 spin_unlock(&inode->i_lock);
Peng Taoacff58802011-07-30 20:52:31 -04001682 dprintk("%s: lseg %p end_pos %llu\n",
Fred Isamancd841602012-04-20 14:47:44 -04001683 __func__, hdr->lseg, nfsi->layout->plh_lwb);
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001684
1685 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one
1686 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
1687 if (mark_as_dirty)
Fred Isamancd841602012-04-20 14:47:44 -04001688 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001689}
1690EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
1691
Andy Adamsondb29c082011-07-30 20:52:38 -04001692void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
1693{
1694 struct nfs_server *nfss = NFS_SERVER(data->args.inode);
1695
1696 if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
1697 nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
1698}
1699
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001700/*
1701 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
1702 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
1703 * data to disk to allow the server to recover the data if it crashes.
1704 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
1705 * is off, and a COMMIT is sent to a data server, or
1706 * if WRITEs to a data server return NFS_DATA_SYNC.
1707 */
Andy Adamson863a3c62011-03-23 13:27:54 +00001708int
Andy Adamsonef311532011-03-12 02:58:10 -05001709pnfs_layoutcommit_inode(struct inode *inode, bool sync)
Andy Adamson863a3c62011-03-23 13:27:54 +00001710{
1711 struct nfs4_layoutcommit_data *data;
1712 struct nfs_inode *nfsi = NFS_I(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001713 loff_t end_pos;
1714 int status = 0;
1715
1716 dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
1717
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001718 if (!test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
1719 return 0;
1720
Andy Adamson863a3c62011-03-23 13:27:54 +00001721 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
1722 data = kzalloc(sizeof(*data), GFP_NOFS);
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001723 if (!data) {
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001724 status = -ENOMEM;
1725 goto out;
1726 }
Andy Adamson863a3c62011-03-23 13:27:54 +00001727
Peng Tao92407e72011-10-23 20:21:17 -07001728 if (!test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
1729 goto out_free;
1730
1731 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
1732 if (!sync) {
1733 status = -EAGAIN;
1734 goto out_free;
1735 }
1736 status = wait_on_bit_lock(&nfsi->flags, NFS_INO_LAYOUTCOMMITTING,
1737 nfs_wait_bit_killable, TASK_KILLABLE);
1738 if (status)
1739 goto out_free;
1740 }
1741
Peng Taoa9bae562011-07-30 20:52:33 -04001742 INIT_LIST_HEAD(&data->lseg_list);
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001743 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00001744 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Peng Tao92407e72011-10-23 20:21:17 -07001745 clear_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags);
Andy Adamson863a3c62011-03-23 13:27:54 +00001746 spin_unlock(&inode->i_lock);
Peng Tao92407e72011-10-23 20:21:17 -07001747 wake_up_bit(&nfsi->flags, NFS_INO_LAYOUTCOMMITTING);
1748 goto out_free;
Andy Adamson863a3c62011-03-23 13:27:54 +00001749 }
Peng Taoa9bae562011-07-30 20:52:33 -04001750
1751 pnfs_list_write_lseg(inode, &data->lseg_list);
Andy Adamson863a3c62011-03-23 13:27:54 +00001752
Peng Taoacff58802011-07-30 20:52:31 -04001753 end_pos = nfsi->layout->plh_lwb;
Peng Taoacff58802011-07-30 20:52:31 -04001754 nfsi->layout->plh_lwb = 0;
Andy Adamson863a3c62011-03-23 13:27:54 +00001755
Trond Myklebustf597c532012-03-04 18:13:56 -05001756 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
Andy Adamson863a3c62011-03-23 13:27:54 +00001757 spin_unlock(&inode->i_lock);
1758
1759 data->args.inode = inode;
Peng Tao9fa40752011-07-30 20:52:32 -04001760 data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
Andy Adamson863a3c62011-03-23 13:27:54 +00001761 nfs_fattr_init(&data->fattr);
1762 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
1763 data->res.fattr = &data->fattr;
1764 data->args.lastbytewritten = end_pos - 1;
1765 data->res.server = NFS_SERVER(inode);
1766
1767 status = nfs4_proc_layoutcommit(data, sync);
1768out:
Peng Tao92407e72011-10-23 20:21:17 -07001769 if (status)
1770 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001771 dprintk("<-- %s status %d\n", __func__, status);
1772 return status;
Peng Tao92407e72011-10-23 20:21:17 -07001773out_free:
1774 kfree(data);
1775 goto out;
Andy Adamson863a3c62011-03-23 13:27:54 +00001776}
Andy Adamson82be4172012-05-23 05:02:35 -04001777
1778struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
1779{
1780 struct nfs4_threshold *thp;
1781
1782 thp = kzalloc(sizeof(*thp), GFP_NOFS);
1783 if (!thp) {
1784 dprintk("%s mdsthreshold allocation failed\n", __func__);
1785 return NULL;
1786 }
1787 return thp;
1788}