blob: b98cd9d84435fe15ea1cb202850508f83b83204b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Milan Broz784aae72009-01-06 03:05:12 +00003 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +01009#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070010
11#include <linux/init.h>
12#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080013#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/moduleparam.h>
15#include <linux/blkpg.h>
16#include <linux/bio.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/mempool.h>
18#include <linux/slab.h>
19#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080020#include <linux/hdreg.h>
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +010021#include <linux/delay.h>
Mike Snitzerffcc3932014-10-28 18:34:52 -040022#include <linux/wait.h>
Li Zefan55782132009-06-09 13:43:05 +080023
24#include <trace/events/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Alasdair G Kergon72d94862006-06-26 00:27:35 -070026#define DM_MSG_PREFIX "core"
27
Namhyung Kim71a16732011-10-31 20:18:54 +000028#ifdef CONFIG_PRINTK
29/*
30 * ratelimit state to be used in DMXXX_LIMIT().
31 */
32DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
33 DEFAULT_RATELIMIT_INTERVAL,
34 DEFAULT_RATELIMIT_BURST);
35EXPORT_SYMBOL(dm_ratelimit_state);
36#endif
37
Milan Broz60935eb2009-06-22 10:12:30 +010038/*
39 * Cookies are numeric values sent with CHANGE and REMOVE
40 * uevents while resuming, removing or renaming the device.
41 */
42#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
43#define DM_COOKIE_LENGTH 24
44
Linus Torvalds1da177e2005-04-16 15:20:36 -070045static const char *_name = DM_NAME;
46
47static unsigned int major = 0;
48static unsigned int _major = 0;
49
Alasdair G Kergond15b7742011-08-02 12:32:01 +010050static DEFINE_IDR(_minor_idr);
51
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070052static DEFINE_SPINLOCK(_minor_lock);
Mikulas Patocka2c140a22013-11-01 18:27:41 -040053
54static void do_deferred_remove(struct work_struct *w);
55
56static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
57
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -040058static struct workqueue_struct *deferred_remove_workqueue;
59
Linus Torvalds1da177e2005-04-16 15:20:36 -070060/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000061 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 * One of these is allocated per bio.
63 */
64struct dm_io {
65 struct mapped_device *md;
66 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010068 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080069 unsigned long start_time;
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +010070 spinlock_t endio_lock;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -040071 struct dm_stats_aux stats_aux;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072};
73
74/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000075 * For request-based dm.
76 * One of these is allocated per request.
77 */
78struct dm_rq_target_io {
79 struct mapped_device *md;
80 struct dm_target *ti;
81 struct request *orig, clone;
82 int error;
83 union map_info info;
84};
85
86/*
Kent Overstreet94818742012-09-07 13:44:01 -070087 * For request-based dm - the bio clones we allocate are embedded in these
88 * structs.
89 *
90 * We allocate these with bio_alloc_bioset, using the front_pad parameter when
91 * the bioset is created - this means the bio has to come at the end of the
92 * struct.
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000093 */
94struct dm_rq_clone_bio_info {
95 struct bio *orig;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +010096 struct dm_rq_target_io *tio;
Kent Overstreet94818742012-09-07 13:44:01 -070097 struct bio clone;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000098};
99
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100100union map_info *dm_get_rq_mapinfo(struct request *rq)
101{
102 if (rq && rq->end_io_data)
103 return &((struct dm_rq_target_io *)rq->end_io_data)->info;
104 return NULL;
105}
106EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
107
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700108#define MINOR_ALLOCED ((void *)-1)
109
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110/*
111 * Bits for the md->flags field.
112 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +0100113#define DMF_BLOCK_IO_FOR_SUSPEND 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -0800115#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700116#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700117#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800118#define DMF_NOFLUSH_SUSPENDING 5
Mikulas Patockad5b9dd02011-08-02 12:32:04 +0100119#define DMF_MERGE_IS_OPTIONAL 6
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400120#define DMF_DEFERRED_REMOVE 7
Mike Snitzerffcc3932014-10-28 18:34:52 -0400121#define DMF_SUSPENDED_INTERNALLY 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122
Milan Broz304f3f62008-02-08 02:11:17 +0000123/*
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100124 * A dummy definition to make RCU happy.
125 * struct dm_table should never be dereferenced in this file.
126 */
127struct dm_table {
128 int undefined__;
129};
130
131/*
Milan Broz304f3f62008-02-08 02:11:17 +0000132 * Work processed by per-device workqueue.
133 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134struct mapped_device {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100135 struct srcu_struct io_barrier;
Daniel Walkere61290a2008-02-08 02:10:08 +0000136 struct mutex suspend_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700138 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
Mikulas Patocka2a7faeb12013-07-10 23:41:18 +0100140 /*
141 * The current mapping.
142 * Use dm_get_live_table{_fast} or take suspend_lock for
143 * dereference.
144 */
Pranith Kumar6fa99522014-10-28 15:09:57 -0700145 struct dm_table __rcu *map;
Mikulas Patocka2a7faeb12013-07-10 23:41:18 +0100146
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500147 struct list_head table_devices;
148 struct mutex table_devices_lock;
149
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 unsigned long flags;
151
Jens Axboe165125e2007-07-24 09:28:11 +0200152 struct request_queue *queue;
Mike Snitzera5664da2010-08-12 04:14:01 +0100153 unsigned type;
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +0100154 /* Protect queue and type against concurrent access. */
Mike Snitzera5664da2010-08-12 04:14:01 +0100155 struct mutex type_lock;
156
Alasdair G Kergon36a04562011-10-31 20:19:04 +0000157 struct target_type *immutable_target_type;
158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -0800160 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
162 void *interface_ptr;
163
164 /*
165 * A list of ios that arrived while we were suspended.
166 */
Nikanth Karthikesan316d3152009-10-06 20:16:55 +0200167 atomic_t pending[2];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 wait_queue_head_t wait;
Mikulas Patocka53d59142009-04-02 19:55:37 +0100169 struct work_struct work;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800170 struct bio_list deferred;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100171 spinlock_t deferred_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
173 /*
Tejun Heo29e40132010-09-08 18:07:00 +0200174 * Processing queue (flush)
Milan Broz304f3f62008-02-08 02:11:17 +0000175 */
176 struct workqueue_struct *wq;
177
178 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 * io objects are allocated from here.
180 */
181 mempool_t *io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182
Stefan Bader9faf4002006-10-03 01:15:41 -0700183 struct bio_set *bs;
184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 /*
186 * Event handling.
187 */
188 atomic_t event_nr;
189 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100190 atomic_t uevent_seq;
191 struct list_head uevent_list;
192 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194 /*
195 * freeze/thaw support require holding onto a super block
196 */
197 struct super_block *frozen_sb;
Mikulas Patockadb8fef42009-06-22 10:12:15 +0100198 struct block_device *bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800199
200 /* forced geometry settings */
201 struct hd_geometry geometry;
Milan Broz784aae72009-01-06 03:05:12 +0000202
Mikulas Patocka2995fa72014-01-13 19:37:54 -0500203 /* kobject and completion */
204 struct dm_kobject_holder kobj_holder;
Mikulas Patockabe35f482014-01-06 23:01:22 -0500205
Tejun Heod87f4c12010-09-03 11:56:19 +0200206 /* zero-length flush that will be cloned and submitted to targets */
207 struct bio flush_bio;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400208
209 struct dm_stats stats;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210};
211
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +0100212/*
213 * For mempools pre-allocation at the table loading time.
214 */
215struct dm_md_mempools {
216 mempool_t *io_pool;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +0100217 struct bio_set *bs;
218};
219
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500220struct table_device {
221 struct list_head list;
222 atomic_t count;
223 struct dm_dev dm_dev;
224};
225
Mike Snitzer6cfa5852013-09-12 18:06:11 -0400226#define RESERVED_BIO_BASED_IOS 16
227#define RESERVED_REQUEST_BASED_IOS 256
Mike Snitzerf4790822013-09-12 18:06:12 -0400228#define RESERVED_MAX_IOS 1024
Christoph Lametere18b8902006-12-06 20:33:20 -0800229static struct kmem_cache *_io_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000230static struct kmem_cache *_rq_tio_cache;
Kent Overstreet94818742012-09-07 13:44:01 -0700231
Mike Snitzerf4790822013-09-12 18:06:12 -0400232/*
Mike Snitzere8603132013-09-12 18:06:12 -0400233 * Bio-based DM's mempools' reserved IOs set by the user.
234 */
235static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
236
237/*
Mike Snitzerf4790822013-09-12 18:06:12 -0400238 * Request-based DM's mempools' reserved IOs set by the user.
239 */
240static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS;
241
242static unsigned __dm_get_reserved_ios(unsigned *reserved_ios,
243 unsigned def, unsigned max)
244{
245 unsigned ios = ACCESS_ONCE(*reserved_ios);
246 unsigned modified_ios = 0;
247
248 if (!ios)
249 modified_ios = def;
250 else if (ios > max)
251 modified_ios = max;
252
253 if (modified_ios) {
254 (void)cmpxchg(reserved_ios, ios, modified_ios);
255 ios = modified_ios;
256 }
257
258 return ios;
259}
260
Mike Snitzere8603132013-09-12 18:06:12 -0400261unsigned dm_get_reserved_bio_based_ios(void)
262{
263 return __dm_get_reserved_ios(&reserved_bio_based_ios,
264 RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS);
265}
266EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
267
Mike Snitzerf4790822013-09-12 18:06:12 -0400268unsigned dm_get_reserved_rq_based_ios(void)
269{
270 return __dm_get_reserved_ios(&reserved_rq_based_ios,
271 RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS);
272}
273EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios);
274
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275static int __init local_init(void)
276{
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100277 int r = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100280 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 if (!_io_cache)
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100282 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000284 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
285 if (!_rq_tio_cache)
Mikulas Patockadba14162012-10-12 21:02:15 +0100286 goto out_free_io_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000287
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100288 r = dm_uevent_init();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100289 if (r)
Jun'ichi Nomura23e50832013-03-01 22:45:48 +0000290 goto out_free_rq_tio_cache;
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100291
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400292 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
293 if (!deferred_remove_workqueue) {
294 r = -ENOMEM;
295 goto out_uevent_exit;
296 }
297
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 _major = major;
299 r = register_blkdev(_major, _name);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100300 if (r < 0)
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400301 goto out_free_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302
303 if (!_major)
304 _major = r;
305
306 return 0;
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100307
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400308out_free_workqueue:
309 destroy_workqueue(deferred_remove_workqueue);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100310out_uevent_exit:
311 dm_uevent_exit();
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000312out_free_rq_tio_cache:
313 kmem_cache_destroy(_rq_tio_cache);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100314out_free_io_cache:
315 kmem_cache_destroy(_io_cache);
316
317 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318}
319
320static void local_exit(void)
321{
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400322 flush_scheduled_work();
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400323 destroy_workqueue(deferred_remove_workqueue);
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400324
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000325 kmem_cache_destroy(_rq_tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700327 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100328 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329
330 _major = 0;
331
332 DMINFO("cleaned up");
333}
334
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000335static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 local_init,
337 dm_target_init,
338 dm_linear_init,
339 dm_stripe_init,
Mikulas Patocka952b3552009-12-10 23:51:57 +0000340 dm_io_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100341 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 dm_interface_init,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400343 dm_statistics_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344};
345
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000346static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 local_exit,
348 dm_target_exit,
349 dm_linear_exit,
350 dm_stripe_exit,
Mikulas Patocka952b3552009-12-10 23:51:57 +0000351 dm_io_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100352 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 dm_interface_exit,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400354 dm_statistics_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355};
356
357static int __init dm_init(void)
358{
359 const int count = ARRAY_SIZE(_inits);
360
361 int r, i;
362
363 for (i = 0; i < count; i++) {
364 r = _inits[i]();
365 if (r)
366 goto bad;
367 }
368
369 return 0;
370
371 bad:
372 while (i--)
373 _exits[i]();
374
375 return r;
376}
377
378static void __exit dm_exit(void)
379{
380 int i = ARRAY_SIZE(_exits);
381
382 while (i--)
383 _exits[i]();
Alasdair G Kergond15b7742011-08-02 12:32:01 +0100384
385 /*
386 * Should be empty by this point.
387 */
Alasdair G Kergond15b7742011-08-02 12:32:01 +0100388 idr_destroy(&_minor_idr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389}
390
391/*
392 * Block device functions
393 */
Mike Anderson432a2122009-12-10 23:52:20 +0000394int dm_deleting_md(struct mapped_device *md)
395{
396 return test_bit(DMF_DELETING, &md->flags);
397}
398
Al Virofe5f9f22008-03-02 10:29:31 -0500399static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400{
401 struct mapped_device *md;
402
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700403 spin_lock(&_minor_lock);
404
Al Virofe5f9f22008-03-02 10:29:31 -0500405 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700406 if (!md)
407 goto out;
408
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700409 if (test_bit(DMF_FREEING, &md->flags) ||
Mike Anderson432a2122009-12-10 23:52:20 +0000410 dm_deleting_md(md)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700411 md = NULL;
412 goto out;
413 }
414
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700416 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700417
418out:
419 spin_unlock(&_minor_lock);
420
421 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422}
423
Al Virodb2a1442013-05-05 21:52:57 -0400424static void dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425{
Al Virofe5f9f22008-03-02 10:29:31 -0500426 struct mapped_device *md = disk->private_data;
Arnd Bergmann6e9624b2010-08-07 18:25:34 +0200427
Milan Broz4a1aeb92011-01-13 19:59:48 +0000428 spin_lock(&_minor_lock);
429
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400430 if (atomic_dec_and_test(&md->open_count) &&
431 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400432 queue_work(deferred_remove_workqueue, &deferred_remove_work);
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400433
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 dm_put(md);
Milan Broz4a1aeb92011-01-13 19:59:48 +0000435
436 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437}
438
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700439int dm_open_count(struct mapped_device *md)
440{
441 return atomic_read(&md->open_count);
442}
443
444/*
445 * Guarantees nothing is using the device before it's deleted.
446 */
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400447int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700448{
449 int r = 0;
450
451 spin_lock(&_minor_lock);
452
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400453 if (dm_open_count(md)) {
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700454 r = -EBUSY;
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400455 if (mark_deferred)
456 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
457 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
458 r = -EEXIST;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700459 else
460 set_bit(DMF_DELETING, &md->flags);
461
462 spin_unlock(&_minor_lock);
463
464 return r;
465}
466
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400467int dm_cancel_deferred_remove(struct mapped_device *md)
468{
469 int r = 0;
470
471 spin_lock(&_minor_lock);
472
473 if (test_bit(DMF_DELETING, &md->flags))
474 r = -EBUSY;
475 else
476 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
477
478 spin_unlock(&_minor_lock);
479
480 return r;
481}
482
483static void do_deferred_remove(struct work_struct *w)
484{
485 dm_deferred_remove();
486}
487
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400488sector_t dm_get_size(struct mapped_device *md)
489{
490 return get_capacity(md->disk);
491}
492
Mike Snitzer9974fa22014-02-28 15:33:43 +0100493struct request_queue *dm_get_md_queue(struct mapped_device *md)
494{
495 return md->queue;
496}
497
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400498struct dm_stats *dm_get_stats(struct mapped_device *md)
499{
500 return &md->stats;
501}
502
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800503static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
504{
505 struct mapped_device *md = bdev->bd_disk->private_data;
506
507 return dm_get_geometry(md, geo);
508}
509
Al Virofe5f9f22008-03-02 10:29:31 -0500510static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
Milan Brozaa129a22006-10-03 01:15:15 -0700511 unsigned int cmd, unsigned long arg)
512{
Al Virofe5f9f22008-03-02 10:29:31 -0500513 struct mapped_device *md = bdev->bd_disk->private_data;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100514 int srcu_idx;
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100515 struct dm_table *map;
Milan Brozaa129a22006-10-03 01:15:15 -0700516 struct dm_target *tgt;
517 int r = -ENOTTY;
518
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100519retry:
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100520 map = dm_get_live_table(md, &srcu_idx);
521
Milan Brozaa129a22006-10-03 01:15:15 -0700522 if (!map || !dm_table_get_size(map))
523 goto out;
524
525 /* We only support devices that have a single target */
526 if (dm_table_get_num_targets(map) != 1)
527 goto out;
528
529 tgt = dm_table_get_target(map, 0);
Mike Snitzer4d341d82014-11-16 14:21:47 -0500530 if (!tgt->type->ioctl)
531 goto out;
Milan Brozaa129a22006-10-03 01:15:15 -0700532
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +0000533 if (dm_suspended_md(md)) {
Milan Brozaa129a22006-10-03 01:15:15 -0700534 r = -EAGAIN;
535 goto out;
536 }
537
Mike Snitzer4d341d82014-11-16 14:21:47 -0500538 r = tgt->type->ioctl(tgt, cmd, arg);
Milan Brozaa129a22006-10-03 01:15:15 -0700539
540out:
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100541 dm_put_live_table(md, srcu_idx);
Milan Brozaa129a22006-10-03 01:15:15 -0700542
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100543 if (r == -ENOTCONN) {
544 msleep(10);
545 goto retry;
546 }
547
Milan Brozaa129a22006-10-03 01:15:15 -0700548 return r;
549}
550
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100551static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552{
553 return mempool_alloc(md->io_pool, GFP_NOIO);
554}
555
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100556static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557{
558 mempool_free(io, md->io_pool);
559}
560
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100561static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562{
Mikulas Patockadba14162012-10-12 21:02:15 +0100563 bio_put(&tio->clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564}
565
Kiyoshi Ueda08885642009-12-10 23:52:15 +0000566static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md,
567 gfp_t gfp_mask)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100568{
Jun'ichi Nomura5f015202013-03-01 22:45:48 +0000569 return mempool_alloc(md->io_pool, gfp_mask);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100570}
571
572static void free_rq_tio(struct dm_rq_target_io *tio)
573{
Jun'ichi Nomura5f015202013-03-01 22:45:48 +0000574 mempool_free(tio, tio->md->io_pool);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100575}
576
Kiyoshi Ueda90abb8c2009-12-10 23:52:13 +0000577static int md_in_flight(struct mapped_device *md)
578{
579 return atomic_read(&md->pending[READ]) +
580 atomic_read(&md->pending[WRITE]);
581}
582
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800583static void start_io_acct(struct dm_io *io)
584{
585 struct mapped_device *md = io->md;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400586 struct bio *bio = io->bio;
Tejun Heoc9959052008-08-25 19:47:21 +0900587 int cpu;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400588 int rw = bio_data_dir(bio);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800589
590 io->start_time = jiffies;
591
Tejun Heo074a7ac2008-08-25 19:56:14 +0900592 cpu = part_stat_lock();
593 part_round_stats(cpu, &dm_disk(md)->part0);
594 part_stat_unlock();
Shaohua Li1e9bb882011-03-22 08:35:35 +0100595 atomic_set(&dm_disk(md)->part0.in_flight[rw],
596 atomic_inc_return(&md->pending[rw]));
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400597
598 if (unlikely(dm_stats_used(&md->stats)))
Kent Overstreet4f024f32013-10-11 15:44:27 -0700599 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400600 bio_sectors(bio), false, 0, &io->stats_aux);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800601}
602
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000603static void end_io_acct(struct dm_io *io)
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800604{
605 struct mapped_device *md = io->md;
606 struct bio *bio = io->bio;
607 unsigned long duration = jiffies - io->start_time;
Gu Zheng18c0b222014-11-24 11:05:26 +0800608 int pending;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800609 int rw = bio_data_dir(bio);
610
Gu Zheng18c0b222014-11-24 11:05:26 +0800611 generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800612
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400613 if (unlikely(dm_stats_used(&md->stats)))
Kent Overstreet4f024f32013-10-11 15:44:27 -0700614 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400615 bio_sectors(bio), true, duration, &io->stats_aux);
616
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100617 /*
618 * After this is decremented the bio must not be touched if it is
Tejun Heod87f4c12010-09-03 11:56:19 +0200619 * a flush.
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100620 */
Shaohua Li1e9bb882011-03-22 08:35:35 +0100621 pending = atomic_dec_return(&md->pending[rw]);
622 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
Nikanth Karthikesan316d3152009-10-06 20:16:55 +0200623 pending += atomic_read(&md->pending[rw^0x1]);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800624
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000625 /* nudge anyone waiting on suspend queue */
626 if (!pending)
627 wake_up(&md->wait);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800628}
629
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630/*
631 * Add the bio to the list of deferred io.
632 */
Mikulas Patocka92c63902009-04-09 00:27:15 +0100633static void queue_io(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200635 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200637 spin_lock_irqsave(&md->deferred_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 bio_list_add(&md->deferred, bio);
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200639 spin_unlock_irqrestore(&md->deferred_lock, flags);
Tejun Heo6a8736d2010-09-08 18:07:00 +0200640 queue_work(md->wq, &md->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641}
642
643/*
644 * Everyone (including functions in this file), should use this
645 * function to access the md->map field, and make sure they call
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100646 * dm_put_live_table() when finished.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 */
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100648struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649{
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100650 *srcu_idx = srcu_read_lock(&md->io_barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100652 return srcu_dereference(md->map, &md->io_barrier);
653}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100655void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
656{
657 srcu_read_unlock(&md->io_barrier, srcu_idx);
658}
659
660void dm_sync_table(struct mapped_device *md)
661{
662 synchronize_srcu(&md->io_barrier);
663 synchronize_rcu_expedited();
664}
665
666/*
667 * A fast alternative to dm_get_live_table/dm_put_live_table.
668 * The caller must not block between these two functions.
669 */
670static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
671{
672 rcu_read_lock();
673 return rcu_dereference(md->map);
674}
675
676static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
677{
678 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679}
680
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800681/*
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500682 * Open a table device so we can use it as a map destination.
683 */
684static int open_table_device(struct table_device *td, dev_t dev,
685 struct mapped_device *md)
686{
687 static char *_claim_ptr = "I belong to device-mapper";
688 struct block_device *bdev;
689
690 int r;
691
692 BUG_ON(td->dm_dev.bdev);
693
694 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
695 if (IS_ERR(bdev))
696 return PTR_ERR(bdev);
697
698 r = bd_link_disk_holder(bdev, dm_disk(md));
699 if (r) {
700 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
701 return r;
702 }
703
704 td->dm_dev.bdev = bdev;
705 return 0;
706}
707
708/*
709 * Close a table device that we've been using.
710 */
711static void close_table_device(struct table_device *td, struct mapped_device *md)
712{
713 if (!td->dm_dev.bdev)
714 return;
715
716 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
717 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
718 td->dm_dev.bdev = NULL;
719}
720
721static struct table_device *find_table_device(struct list_head *l, dev_t dev,
722 fmode_t mode) {
723 struct table_device *td;
724
725 list_for_each_entry(td, l, list)
726 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
727 return td;
728
729 return NULL;
730}
731
732int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
733 struct dm_dev **result) {
734 int r;
735 struct table_device *td;
736
737 mutex_lock(&md->table_devices_lock);
738 td = find_table_device(&md->table_devices, dev, mode);
739 if (!td) {
740 td = kmalloc(sizeof(*td), GFP_KERNEL);
741 if (!td) {
742 mutex_unlock(&md->table_devices_lock);
743 return -ENOMEM;
744 }
745
746 td->dm_dev.mode = mode;
747 td->dm_dev.bdev = NULL;
748
749 if ((r = open_table_device(td, dev, md))) {
750 mutex_unlock(&md->table_devices_lock);
751 kfree(td);
752 return r;
753 }
754
755 format_dev_t(td->dm_dev.name, dev);
756
757 atomic_set(&td->count, 0);
758 list_add(&td->list, &md->table_devices);
759 }
760 atomic_inc(&td->count);
761 mutex_unlock(&md->table_devices_lock);
762
763 *result = &td->dm_dev;
764 return 0;
765}
766EXPORT_SYMBOL_GPL(dm_get_table_device);
767
768void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
769{
770 struct table_device *td = container_of(d, struct table_device, dm_dev);
771
772 mutex_lock(&md->table_devices_lock);
773 if (atomic_dec_and_test(&td->count)) {
774 close_table_device(td, md);
775 list_del(&td->list);
776 kfree(td);
777 }
778 mutex_unlock(&md->table_devices_lock);
779}
780EXPORT_SYMBOL(dm_put_table_device);
781
782static void free_table_devices(struct list_head *devices)
783{
784 struct list_head *tmp, *next;
785
786 list_for_each_safe(tmp, next, devices) {
787 struct table_device *td = list_entry(tmp, struct table_device, list);
788
789 DMWARN("dm_destroy: %s still exists with %d references",
790 td->dm_dev.name, atomic_read(&td->count));
791 kfree(td);
792 }
793}
794
795/*
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800796 * Get the geometry associated with a dm device
797 */
798int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
799{
800 *geo = md->geometry;
801
802 return 0;
803}
804
805/*
806 * Set the geometry of a device.
807 */
808int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
809{
810 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
811
812 if (geo->start > sz) {
813 DMWARN("Start sector is beyond the geometry limits.");
814 return -EINVAL;
815 }
816
817 md->geometry = *geo;
818
819 return 0;
820}
821
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822/*-----------------------------------------------------------------
823 * CRUD START:
824 * A more elegant soln is in the works that uses the queue
825 * merge fn, unfortunately there are a couple of changes to
826 * the block layer that I want to make for this. So in the
827 * interests of getting something for people to use I give
828 * you this clearly demarcated crap.
829 *---------------------------------------------------------------*/
830
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800831static int __noflush_suspending(struct mapped_device *md)
832{
833 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
834}
835
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836/*
837 * Decrements the number of outstanding ios that a bio has been
838 * cloned into, completing the original io if necc.
839 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800840static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800842 unsigned long flags;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000843 int io_error;
844 struct bio *bio;
845 struct mapped_device *md = io->md;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800846
847 /* Push-back supersedes any I/O errors */
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +0100848 if (unlikely(error)) {
849 spin_lock_irqsave(&io->endio_lock, flags);
850 if (!(io->error > 0 && __noflush_suspending(md)))
851 io->error = error;
852 spin_unlock_irqrestore(&io->endio_lock, flags);
853 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854
855 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800856 if (io->error == DM_ENDIO_REQUEUE) {
857 /*
858 * Target requested pushing back the I/O.
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800859 */
Mikulas Patocka022c2612009-04-02 19:55:39 +0100860 spin_lock_irqsave(&md->deferred_lock, flags);
Tejun Heo6a8736d2010-09-08 18:07:00 +0200861 if (__noflush_suspending(md))
862 bio_list_add_head(&md->deferred, io->bio);
863 else
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800864 /* noflush suspend was interrupted. */
865 io->error = -EIO;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100866 spin_unlock_irqrestore(&md->deferred_lock, flags);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800867 }
868
Milan Brozb35f8ca2009-03-16 17:44:36 +0000869 io_error = io->error;
870 bio = io->bio;
Tejun Heo6a8736d2010-09-08 18:07:00 +0200871 end_io_acct(io);
872 free_io(md, io);
Jens Axboe2056a782006-03-23 20:00:26 +0100873
Tejun Heo6a8736d2010-09-08 18:07:00 +0200874 if (io_error == DM_ENDIO_REQUEUE)
875 return;
876
Kent Overstreet4f024f32013-10-11 15:44:27 -0700877 if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) {
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100878 /*
Tejun Heo6a8736d2010-09-08 18:07:00 +0200879 * Preflush done for flush with data, reissue
880 * without REQ_FLUSH.
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100881 */
Tejun Heo6a8736d2010-09-08 18:07:00 +0200882 bio->bi_rw &= ~REQ_FLUSH;
883 queue_io(md, bio);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100884 } else {
Mike Snitzerb372d362010-09-08 18:07:01 +0200885 /* done with normal IO or empty flush */
Linus Torvalds0a82a8d2013-04-18 09:00:26 -0700886 trace_block_bio_complete(md->queue, bio, io_error);
Mike Snitzerb372d362010-09-08 18:07:01 +0200887 bio_endio(bio, io_error);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800888 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 }
890}
891
Mike Snitzer7eee4ae2014-06-02 15:50:06 -0400892static void disable_write_same(struct mapped_device *md)
893{
894 struct queue_limits *limits = dm_get_queue_limits(md);
895
896 /* device doesn't really support WRITE SAME, disable it */
897 limits->max_write_same_sectors = 0;
898}
899
NeilBrown6712ecf2007-09-27 12:47:43 +0200900static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901{
zhendong chen5164bec2014-12-17 14:37:04 +0800902 int r = error;
Mikulas Patockabfc6d412014-03-04 18:24:49 -0500903 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000904 struct dm_io *io = tio->io;
Stefan Bader9faf4002006-10-03 01:15:41 -0700905 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 dm_endio_fn endio = tio->ti->type->end_io;
907
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
909 error = -EIO;
910
911 if (endio) {
Mikulas Patocka7de3ee52012-12-21 20:23:41 +0000912 r = endio(tio->ti, bio, error);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800913 if (r < 0 || r == DM_ENDIO_REQUEUE)
914 /*
915 * error and requeue request are handled
916 * in dec_pending().
917 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800919 else if (r == DM_ENDIO_INCOMPLETE)
920 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200921 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800922 else if (r) {
923 DMWARN("unimplemented target endio return value: %d", r);
924 BUG();
925 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 }
927
Mike Snitzer7eee4ae2014-06-02 15:50:06 -0400928 if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) &&
929 !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors))
930 disable_write_same(md);
931
Stefan Bader9faf4002006-10-03 01:15:41 -0700932 free_tio(md, tio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000933 dec_pending(io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934}
935
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100936/*
937 * Partial completion handling for request-based dm
938 */
939static void end_clone_bio(struct bio *clone, int error)
940{
Mikulas Patockabfc6d412014-03-04 18:24:49 -0500941 struct dm_rq_clone_bio_info *info =
942 container_of(clone, struct dm_rq_clone_bio_info, clone);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100943 struct dm_rq_target_io *tio = info->tio;
944 struct bio *bio = info->orig;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700945 unsigned int nr_bytes = info->orig->bi_iter.bi_size;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100946
947 bio_put(clone);
948
949 if (tio->error)
950 /*
951 * An error has already been detected on the request.
952 * Once error occurred, just let clone->end_io() handle
953 * the remainder.
954 */
955 return;
956 else if (error) {
957 /*
958 * Don't notice the error to the upper layer yet.
959 * The error handling decision is made by the target driver,
960 * when the request is completed.
961 */
962 tio->error = error;
963 return;
964 }
965
966 /*
967 * I/O for the bio successfully completed.
968 * Notice the data completion to the upper layer.
969 */
970
971 /*
972 * bios are processed from the head of the list.
973 * So the completing bio should always be rq->bio.
974 * If it's not, something wrong is happening.
975 */
976 if (tio->orig->bio != bio)
977 DMERR("bio completion is going in the middle of the request");
978
979 /*
980 * Update the original request.
981 * Do not use blk_end_request() here, because it may complete
982 * the original request before the clone, and break the ordering.
983 */
984 blk_update_request(tio->orig, 0, nr_bytes);
985}
986
987/*
988 * Don't touch any member of the md after calling this function because
989 * the md may be freed in dm_put() at the end of this function.
990 * Or do dm_get() before calling this function and dm_put() later.
991 */
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +0000992static void rq_completed(struct mapped_device *md, int rw, int run_queue)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100993{
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +0000994 atomic_dec(&md->pending[rw]);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100995
996 /* nudge anyone waiting on suspend queue */
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +0000997 if (!md_in_flight(md))
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100998 wake_up(&md->wait);
999
Jens Axboea8c32a52012-11-06 12:24:26 +01001000 /*
1001 * Run this off this callpath, as drivers could invoke end_io while
1002 * inside their request_fn (and holding the queue lock). Calling
1003 * back into ->request_fn() could deadlock attempting to grab the
1004 * queue lock again.
1005 */
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001006 if (run_queue)
Jens Axboea8c32a52012-11-06 12:24:26 +01001007 blk_run_queue_async(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001008
1009 /*
1010 * dm_put() must be at the end of this function. See the comment above
1011 */
1012 dm_put(md);
1013}
1014
Kiyoshi Uedaa77e28c2009-09-04 20:40:16 +01001015static void free_rq_clone(struct request *clone)
1016{
1017 struct dm_rq_target_io *tio = clone->end_io_data;
1018
1019 blk_rq_unprep_clone(clone);
1020 free_rq_tio(tio);
1021}
1022
Kiyoshi Ueda980691e2009-12-10 23:52:17 +00001023/*
1024 * Complete the clone and the original request.
1025 * Must be called without queue lock.
1026 */
1027static void dm_end_request(struct request *clone, int error)
1028{
1029 int rw = rq_data_dir(clone);
1030 struct dm_rq_target_io *tio = clone->end_io_data;
1031 struct mapped_device *md = tio->md;
1032 struct request *rq = tio->orig;
1033
Tejun Heo29e40132010-09-08 18:07:00 +02001034 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
Kiyoshi Ueda980691e2009-12-10 23:52:17 +00001035 rq->errors = clone->errors;
1036 rq->resid_len = clone->resid_len;
1037
1038 if (rq->sense)
1039 /*
1040 * We are using the sense buffer of the original
1041 * request.
1042 * So setting the length of the sense data is enough.
1043 */
1044 rq->sense_len = clone->sense_len;
1045 }
1046
1047 free_rq_clone(clone);
Tejun Heo29e40132010-09-08 18:07:00 +02001048 blk_end_request_all(rq, error);
1049 rq_completed(md, rw, true);
Kiyoshi Ueda980691e2009-12-10 23:52:17 +00001050}
1051
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001052static void dm_unprep_request(struct request *rq)
1053{
1054 struct request *clone = rq->special;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001055
1056 rq->special = NULL;
1057 rq->cmd_flags &= ~REQ_DONTPREP;
1058
Kiyoshi Uedaa77e28c2009-09-04 20:40:16 +01001059 free_rq_clone(clone);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001060}
1061
1062/*
1063 * Requeue the original request of a clone.
1064 */
1065void dm_requeue_unmapped_request(struct request *clone)
1066{
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00001067 int rw = rq_data_dir(clone);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001068 struct dm_rq_target_io *tio = clone->end_io_data;
1069 struct mapped_device *md = tio->md;
1070 struct request *rq = tio->orig;
1071 struct request_queue *q = rq->q;
1072 unsigned long flags;
1073
1074 dm_unprep_request(rq);
1075
1076 spin_lock_irqsave(q->queue_lock, flags);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001077 blk_requeue_request(q, rq);
1078 spin_unlock_irqrestore(q->queue_lock, flags);
1079
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00001080 rq_completed(md, rw, 0);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001081}
1082EXPORT_SYMBOL_GPL(dm_requeue_unmapped_request);
1083
1084static void __stop_queue(struct request_queue *q)
1085{
1086 blk_stop_queue(q);
1087}
1088
1089static void stop_queue(struct request_queue *q)
1090{
1091 unsigned long flags;
1092
1093 spin_lock_irqsave(q->queue_lock, flags);
1094 __stop_queue(q);
1095 spin_unlock_irqrestore(q->queue_lock, flags);
1096}
1097
1098static void __start_queue(struct request_queue *q)
1099{
1100 if (blk_queue_stopped(q))
1101 blk_start_queue(q);
1102}
1103
1104static void start_queue(struct request_queue *q)
1105{
1106 unsigned long flags;
1107
1108 spin_lock_irqsave(q->queue_lock, flags);
1109 __start_queue(q);
1110 spin_unlock_irqrestore(q->queue_lock, flags);
1111}
1112
Kiyoshi Ueda11a68242009-12-10 23:52:17 +00001113static void dm_done(struct request *clone, int error, bool mapped)
1114{
1115 int r = error;
1116 struct dm_rq_target_io *tio = clone->end_io_data;
Mike Snitzerba1cbad2012-09-26 23:45:42 +01001117 dm_request_endio_fn rq_end_io = NULL;
Kiyoshi Ueda11a68242009-12-10 23:52:17 +00001118
Mike Snitzerba1cbad2012-09-26 23:45:42 +01001119 if (tio->ti) {
1120 rq_end_io = tio->ti->type->rq_end_io;
1121
1122 if (mapped && rq_end_io)
1123 r = rq_end_io(tio->ti, clone, error, &tio->info);
1124 }
Kiyoshi Ueda11a68242009-12-10 23:52:17 +00001125
Mike Snitzer7eee4ae2014-06-02 15:50:06 -04001126 if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) &&
1127 !clone->q->limits.max_write_same_sectors))
1128 disable_write_same(tio->md);
1129
Kiyoshi Ueda11a68242009-12-10 23:52:17 +00001130 if (r <= 0)
1131 /* The target wants to complete the I/O */
1132 dm_end_request(clone, r);
1133 else if (r == DM_ENDIO_INCOMPLETE)
1134 /* The target will handle the I/O */
1135 return;
1136 else if (r == DM_ENDIO_REQUEUE)
1137 /* The target wants to requeue the I/O */
1138 dm_requeue_unmapped_request(clone);
1139 else {
1140 DMWARN("unimplemented target endio return value: %d", r);
1141 BUG();
1142 }
1143}
1144
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001145/*
1146 * Request completion handler for request-based dm
1147 */
1148static void dm_softirq_done(struct request *rq)
1149{
Kiyoshi Ueda11a68242009-12-10 23:52:17 +00001150 bool mapped = true;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001151 struct request *clone = rq->completion_data;
1152 struct dm_rq_target_io *tio = clone->end_io_data;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001153
Kiyoshi Ueda11a68242009-12-10 23:52:17 +00001154 if (rq->cmd_flags & REQ_FAILED)
1155 mapped = false;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001156
Kiyoshi Ueda11a68242009-12-10 23:52:17 +00001157 dm_done(clone, tio->error, mapped);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001158}
1159
1160/*
1161 * Complete the clone and the original request with the error status
1162 * through softirq context.
1163 */
1164static void dm_complete_request(struct request *clone, int error)
1165{
1166 struct dm_rq_target_io *tio = clone->end_io_data;
1167 struct request *rq = tio->orig;
1168
1169 tio->error = error;
1170 rq->completion_data = clone;
1171 blk_complete_request(rq);
1172}
1173
1174/*
1175 * Complete the not-mapped clone and the original request with the error status
1176 * through softirq context.
1177 * Target's rq_end_io() function isn't called.
1178 * This may be used when the target's map_rq() function fails.
1179 */
1180void dm_kill_unmapped_request(struct request *clone, int error)
1181{
1182 struct dm_rq_target_io *tio = clone->end_io_data;
1183 struct request *rq = tio->orig;
1184
1185 rq->cmd_flags |= REQ_FAILED;
1186 dm_complete_request(clone, error);
1187}
1188EXPORT_SYMBOL_GPL(dm_kill_unmapped_request);
1189
1190/*
1191 * Called with the queue lock held
1192 */
1193static void end_clone_request(struct request *clone, int error)
1194{
1195 /*
1196 * For just cleaning up the information of the queue in which
1197 * the clone was dispatched.
1198 * The clone is *NOT* freed actually here because it is alloced from
1199 * dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
1200 */
1201 __blk_put_request(clone->q, clone);
1202
1203 /*
1204 * Actual request completion is done in a softirq context which doesn't
1205 * hold the queue lock. Otherwise, deadlock could occur because:
1206 * - another request may be submitted by the upper level driver
1207 * of the stacking during the completion
1208 * - the submission which requires queue lock may be done
1209 * against this queue
1210 */
1211 dm_complete_request(clone, error);
1212}
1213
Mike Snitzer56a67df2010-08-12 04:14:10 +01001214/*
1215 * Return maximum size of I/O possible at the supplied sector up to the current
1216 * target boundary.
1217 */
1218static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219{
Mike Snitzer56a67df2010-08-12 04:14:10 +01001220 sector_t target_offset = dm_target_offset(ti, sector);
1221
1222 return ti->len - target_offset;
1223}
1224
1225static sector_t max_io_len(sector_t sector, struct dm_target *ti)
1226{
1227 sector_t len = max_io_len_target_boundary(sector, ti);
Mike Snitzer542f9032012-07-27 15:08:00 +01001228 sector_t offset, max_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001229
1230 /*
Mike Snitzer542f9032012-07-27 15:08:00 +01001231 * Does the target need to split even further?
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 */
Mike Snitzer542f9032012-07-27 15:08:00 +01001233 if (ti->max_io_len) {
1234 offset = dm_target_offset(ti, sector);
1235 if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
1236 max_len = sector_div(offset, ti->max_io_len);
1237 else
1238 max_len = offset & (ti->max_io_len - 1);
1239 max_len = ti->max_io_len - max_len;
1240
1241 if (len > max_len)
1242 len = max_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 }
1244
1245 return len;
1246}
1247
Mike Snitzer542f9032012-07-27 15:08:00 +01001248int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1249{
1250 if (len > UINT_MAX) {
1251 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1252 (unsigned long long)len, UINT_MAX);
1253 ti->error = "Maximum size of target IO is too large";
1254 return -EINVAL;
1255 }
1256
1257 ti->max_io_len = (uint32_t) len;
1258
1259 return 0;
1260}
1261EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1262
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001263/*
1264 * A target may call dm_accept_partial_bio only from the map routine. It is
1265 * allowed for all bio types except REQ_FLUSH.
1266 *
1267 * dm_accept_partial_bio informs the dm that the target only wants to process
1268 * additional n_sectors sectors of the bio and the rest of the data should be
1269 * sent in a next bio.
1270 *
1271 * A diagram that explains the arithmetics:
1272 * +--------------------+---------------+-------+
1273 * | 1 | 2 | 3 |
1274 * +--------------------+---------------+-------+
1275 *
1276 * <-------------- *tio->len_ptr --------------->
1277 * <------- bi_size ------->
1278 * <-- n_sectors -->
1279 *
1280 * Region 1 was already iterated over with bio_advance or similar function.
1281 * (it may be empty if the target doesn't use bio_advance)
1282 * Region 2 is the remaining bio size that the target wants to process.
1283 * (it may be empty if region 1 is non-empty, although there is no reason
1284 * to make it empty)
1285 * The target requires that region 3 is to be sent in the next bio.
1286 *
1287 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1288 * the partially processed part (the sum of regions 1+2) must be the same for all
1289 * copies of the bio.
1290 */
1291void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1292{
1293 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1294 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1295 BUG_ON(bio->bi_rw & REQ_FLUSH);
1296 BUG_ON(bi_size > *tio->len_ptr);
1297 BUG_ON(n_sectors > bi_size);
1298 *tio->len_ptr -= bi_size - n_sectors;
1299 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1300}
1301EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1302
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001303static void __map_bio(struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304{
1305 int r;
Jens Axboe2056a782006-03-23 20:00:26 +01001306 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -07001307 struct mapped_device *md;
Mikulas Patockadba14162012-10-12 21:02:15 +01001308 struct bio *clone = &tio->clone;
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001309 struct dm_target *ti = tio->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 clone->bi_end_io = clone_endio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
1313 /*
1314 * Map the clone. If r == 0 we don't need to do
1315 * anything, the target has assumed ownership of
1316 * this io.
1317 */
1318 atomic_inc(&tio->io->io_count);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001319 sector = clone->bi_iter.bi_sector;
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00001320 r = ti->type->map(ti, clone);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -08001321 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +01001323
Mike Snitzerd07335e2010-11-16 12:52:38 +01001324 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
1325 tio->io->bio->bi_bdev->bd_dev, sector);
Jens Axboe2056a782006-03-23 20:00:26 +01001326
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001328 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
1329 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -07001330 md = tio->io->md;
1331 dec_pending(tio->io, r);
Stefan Bader9faf4002006-10-03 01:15:41 -07001332 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -08001333 } else if (r) {
1334 DMWARN("unimplemented target map return value: %d", r);
1335 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 }
1337}
1338
1339struct clone_info {
1340 struct mapped_device *md;
1341 struct dm_table *map;
1342 struct bio *bio;
1343 struct dm_io *io;
1344 sector_t sector;
Mikulas Patockae0d66092014-03-14 18:40:39 -04001345 unsigned sector_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346};
1347
Mikulas Patockae0d66092014-03-14 18:40:39 -04001348static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001349{
Kent Overstreet4f024f32013-10-11 15:44:27 -07001350 bio->bi_iter.bi_sector = sector;
1351 bio->bi_iter.bi_size = to_bytes(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352}
1353
1354/*
1355 * Creates a bio that consists of range of complete bvecs.
1356 */
Mikulas Patockadba14162012-10-12 21:02:15 +01001357static void clone_bio(struct dm_target_io *tio, struct bio *bio,
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001358 sector_t sector, unsigned len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359{
Mikulas Patockadba14162012-10-12 21:02:15 +01001360 struct bio *clone = &tio->clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001362 __bio_clone_fast(clone, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001364 if (bio_integrity(bio))
1365 bio_integrity_clone(clone, bio, GFP_NOIO);
1366
1367 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1368 clone->bi_iter.bi_size = to_bytes(len);
1369
1370 if (bio_integrity(bio))
1371 bio_integrity_trim(clone, 0, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001372}
1373
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001374static struct dm_target_io *alloc_tio(struct clone_info *ci,
Junichi Nomura99778272014-10-03 11:55:16 +00001375 struct dm_target *ti,
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001376 unsigned target_bio_nr)
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001377{
Mikulas Patockadba14162012-10-12 21:02:15 +01001378 struct dm_target_io *tio;
1379 struct bio *clone;
1380
Junichi Nomura99778272014-10-03 11:55:16 +00001381 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
Mikulas Patockadba14162012-10-12 21:02:15 +01001382 tio = container_of(clone, struct dm_target_io, clone);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001383
1384 tio->io = ci->io;
1385 tio->ti = ti;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001386 tio->target_bio_nr = target_bio_nr;
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001387
1388 return tio;
1389}
1390
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001391static void __clone_and_map_simple_bio(struct clone_info *ci,
1392 struct dm_target *ti,
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001393 unsigned target_bio_nr, unsigned *len)
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001394{
Junichi Nomura99778272014-10-03 11:55:16 +00001395 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
Mikulas Patockadba14162012-10-12 21:02:15 +01001396 struct bio *clone = &tio->clone;
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001397
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001398 tio->len_ptr = len;
1399
Junichi Nomura99778272014-10-03 11:55:16 +00001400 __bio_clone_fast(clone, ci->bio);
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001401 if (len)
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001402 bio_setup_sector(clone, ci->sector, *len);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001403
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001404 __map_bio(tio);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001405}
1406
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001407static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001408 unsigned num_bios, unsigned *len)
Mike Snitzer06a426c2010-08-12 04:14:09 +01001409{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001410 unsigned target_bio_nr;
Mike Snitzer06a426c2010-08-12 04:14:09 +01001411
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001412 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001413 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
Mike Snitzer06a426c2010-08-12 04:14:09 +01001414}
1415
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001416static int __send_empty_flush(struct clone_info *ci)
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001417{
Mike Snitzer06a426c2010-08-12 04:14:09 +01001418 unsigned target_nr = 0;
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001419 struct dm_target *ti;
1420
Mike Snitzerb372d362010-09-08 18:07:01 +02001421 BUG_ON(bio_has_data(ci->bio));
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001422 while ((ti = dm_table_get_target(ci->map, target_nr++)))
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001423 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001424
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001425 return 0;
1426}
1427
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001428static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001429 sector_t sector, unsigned *len)
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001430{
Mikulas Patockadba14162012-10-12 21:02:15 +01001431 struct bio *bio = ci->bio;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001432 struct dm_target_io *tio;
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001433 unsigned target_bio_nr;
1434 unsigned num_target_bios = 1;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001435
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001436 /*
1437 * Does the target want to receive duplicate copies of the bio?
1438 */
1439 if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1440 num_target_bios = ti->num_write_bios(ti, bio);
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001441
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001442 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
Junichi Nomura99778272014-10-03 11:55:16 +00001443 tio = alloc_tio(ci, ti, target_bio_nr);
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001444 tio->len_ptr = len;
1445 clone_bio(tio, bio, sector, *len);
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001446 __map_bio(tio);
1447 }
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001448}
1449
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001450typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
Mike Snitzer23508a92012-12-21 20:23:37 +00001451
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001452static unsigned get_num_discard_bios(struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001453{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001454 return ti->num_discard_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001455}
1456
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001457static unsigned get_num_write_same_bios(struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001458{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001459 return ti->num_write_same_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001460}
1461
1462typedef bool (*is_split_required_fn)(struct dm_target *ti);
1463
1464static bool is_split_required_for_discard(struct dm_target *ti)
1465{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001466 return ti->split_discard_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001467}
1468
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001469static int __send_changing_extent_only(struct clone_info *ci,
1470 get_num_bios_fn get_num_bios,
1471 is_split_required_fn is_split_required)
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001472{
1473 struct dm_target *ti;
Mikulas Patockae0d66092014-03-14 18:40:39 -04001474 unsigned len;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001475 unsigned num_bios;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001476
Mike Snitzera79245b2010-08-12 04:14:24 +01001477 do {
1478 ti = dm_table_find_target(ci->map, ci->sector);
1479 if (!dm_target_is_valid(ti))
1480 return -EIO;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001481
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001482 /*
Mike Snitzer23508a92012-12-21 20:23:37 +00001483 * Even though the device advertised support for this type of
1484 * request, that does not mean every target supports it, and
Mike Snitzer936688d2011-08-02 12:32:01 +01001485 * reconfiguration might also have changed that since the
Mike Snitzera79245b2010-08-12 04:14:24 +01001486 * check was performed.
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001487 */
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001488 num_bios = get_num_bios ? get_num_bios(ti) : 0;
1489 if (!num_bios)
Mike Snitzera79245b2010-08-12 04:14:24 +01001490 return -EOPNOTSUPP;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001491
Mike Snitzer23508a92012-12-21 20:23:37 +00001492 if (is_split_required && !is_split_required(ti))
Mikulas Patockae0d66092014-03-14 18:40:39 -04001493 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
Mikulas Patocka7acf0272012-07-27 15:08:03 +01001494 else
Mikulas Patockae0d66092014-03-14 18:40:39 -04001495 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
Mike Snitzer06a426c2010-08-12 04:14:09 +01001496
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001497 __send_duplicate_bios(ci, ti, num_bios, &len);
Mike Snitzera79245b2010-08-12 04:14:24 +01001498
1499 ci->sector += len;
1500 } while (ci->sector_count -= len);
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001501
1502 return 0;
1503}
1504
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001505static int __send_discard(struct clone_info *ci)
Mike Snitzer23508a92012-12-21 20:23:37 +00001506{
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001507 return __send_changing_extent_only(ci, get_num_discard_bios,
1508 is_split_required_for_discard);
Mike Snitzer23508a92012-12-21 20:23:37 +00001509}
1510
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001511static int __send_write_same(struct clone_info *ci)
Mike Snitzer23508a92012-12-21 20:23:37 +00001512{
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001513 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
Mike Snitzer23508a92012-12-21 20:23:37 +00001514}
1515
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001516/*
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001517 * Select the correct strategy for processing a non-flush bio.
1518 */
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001519static int __split_and_process_non_flush(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520{
Mikulas Patockadba14162012-10-12 21:02:15 +01001521 struct bio *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001522 struct dm_target *ti;
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001523 unsigned len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001525 if (unlikely(bio->bi_rw & REQ_DISCARD))
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001526 return __send_discard(ci);
Mike Snitzer23508a92012-12-21 20:23:37 +00001527 else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001528 return __send_write_same(ci);
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001529
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001530 ti = dm_table_find_target(ci->map, ci->sector);
1531 if (!dm_target_is_valid(ti))
1532 return -EIO;
1533
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001534 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001535
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001536 __clone_and_map_data_bio(ci, ti, ci->sector, &len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001538 ci->sector += len;
1539 ci->sector_count -= len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001541 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542}
1543
1544/*
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001545 * Entry point to split a bio into clones and submit them to the targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 */
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001547static void __split_and_process_bio(struct mapped_device *md,
1548 struct dm_table *map, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549{
1550 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001551 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001553 if (unlikely(!map)) {
Tejun Heo6a8736d2010-09-08 18:07:00 +02001554 bio_io_error(bio);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +01001555 return;
1556 }
Mikulas Patocka692d0eb2009-04-09 00:27:13 +01001557
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001558 ci.map = map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 ci.md = md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560 ci.io = alloc_io(md);
1561 ci.io->error = 0;
1562 atomic_set(&ci.io->io_count, 1);
1563 ci.io->bio = bio;
1564 ci.io->md = md;
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +01001565 spin_lock_init(&ci.io->endio_lock);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001566 ci.sector = bio->bi_iter.bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -08001568 start_io_acct(ci.io);
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001569
Mike Snitzerb372d362010-09-08 18:07:01 +02001570 if (bio->bi_rw & REQ_FLUSH) {
1571 ci.bio = &ci.md->flush_bio;
1572 ci.sector_count = 0;
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001573 error = __send_empty_flush(&ci);
Mike Snitzerb372d362010-09-08 18:07:01 +02001574 /* dec_pending submits any data associated with flush */
1575 } else {
Tejun Heo6a8736d2010-09-08 18:07:00 +02001576 ci.bio = bio;
Tejun Heod87f4c12010-09-03 11:56:19 +02001577 ci.sector_count = bio_sectors(bio);
Mike Snitzerb372d362010-09-08 18:07:01 +02001578 while (ci.sector_count && !error)
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001579 error = __split_and_process_non_flush(&ci);
Tejun Heod87f4c12010-09-03 11:56:19 +02001580 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581
1582 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001583 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584}
1585/*-----------------------------------------------------------------
1586 * CRUD END
1587 *---------------------------------------------------------------*/
1588
Milan Brozf6fccb12008-07-21 12:00:37 +01001589static int dm_merge_bvec(struct request_queue *q,
1590 struct bvec_merge_data *bvm,
1591 struct bio_vec *biovec)
1592{
1593 struct mapped_device *md = q->queuedata;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001594 struct dm_table *map = dm_get_live_table_fast(md);
Milan Brozf6fccb12008-07-21 12:00:37 +01001595 struct dm_target *ti;
1596 sector_t max_sectors;
Mikulas Patocka50371082008-10-01 14:39:17 +01001597 int max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +01001598
1599 if (unlikely(!map))
Mikulas Patocka50371082008-10-01 14:39:17 +01001600 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +01001601
1602 ti = dm_table_find_target(map, bvm->bi_sector);
Mikulas Patockab01cd5a2008-10-01 14:39:24 +01001603 if (!dm_target_is_valid(ti))
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001604 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +01001605
1606 /*
1607 * Find maximum amount of I/O that won't need splitting
1608 */
Mike Snitzer56a67df2010-08-12 04:14:10 +01001609 max_sectors = min(max_io_len(bvm->bi_sector, ti),
Mike Snitzer148e51b2014-10-09 19:32:22 -04001610 (sector_t) queue_max_sectors(q));
Milan Brozf6fccb12008-07-21 12:00:37 +01001611 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
Mike Snitzer148e51b2014-10-09 19:32:22 -04001612 if (unlikely(max_size < 0)) /* this shouldn't _ever_ happen */
Milan Brozf6fccb12008-07-21 12:00:37 +01001613 max_size = 0;
1614
1615 /*
1616 * merge_bvec_fn() returns number of bytes
1617 * it can accept at this offset
1618 * max is precomputed maximal io size
1619 */
1620 if (max_size && ti->type->merge)
1621 max_size = ti->type->merge(ti, bvm, biovec, max_size);
Mikulas Patocka8cbeb672009-06-22 10:12:14 +01001622 /*
1623 * If the target doesn't support merge method and some of the devices
Mike Snitzer148e51b2014-10-09 19:32:22 -04001624 * provided their merge_bvec method (we know this by looking for the
1625 * max_hw_sectors that dm_set_device_limits may set), then we can't
1626 * allow bios with multiple vector entries. So always set max_size
1627 * to 0, and the code below allows just one page.
Mikulas Patocka8cbeb672009-06-22 10:12:14 +01001628 */
1629 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
Mikulas Patocka8cbeb672009-06-22 10:12:14 +01001630 max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +01001631
Mikulas Patocka50371082008-10-01 14:39:17 +01001632out:
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001633 dm_put_live_table_fast(md);
Milan Brozf6fccb12008-07-21 12:00:37 +01001634 /*
1635 * Always allow an entire first page
1636 */
1637 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1638 max_size = biovec->bv_len;
1639
Milan Brozf6fccb12008-07-21 12:00:37 +01001640 return max_size;
1641}
1642
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643/*
1644 * The request function that just remaps the bio built up by
1645 * dm_merge_bvec.
1646 */
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001647static void _dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648{
Kevin Corry12f03a42006-02-01 03:04:52 -08001649 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650 struct mapped_device *md = q->queuedata;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001651 int srcu_idx;
1652 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001654 map = dm_get_live_table(md, &srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655
Gu Zheng18c0b222014-11-24 11:05:26 +08001656 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);
Kevin Corry12f03a42006-02-01 03:04:52 -08001657
Tejun Heo6a8736d2010-09-08 18:07:00 +02001658 /* if we're suspended, we have to queue this io for later */
1659 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001660 dm_put_live_table(md, srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661
Tejun Heo6a8736d2010-09-08 18:07:00 +02001662 if (bio_rw(bio) != READA)
1663 queue_io(md, bio);
1664 else
Alasdair G Kergon54d9a1b2009-04-09 00:27:14 +01001665 bio_io_error(bio);
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001666 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 }
1668
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001669 __split_and_process_bio(md, map, bio);
1670 dm_put_live_table(md, srcu_idx);
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001671 return;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001672}
1673
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04001674int dm_request_based(struct mapped_device *md)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001675{
1676 return blk_queue_stackable(md->queue);
1677}
1678
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001679static void dm_request(struct request_queue *q, struct bio *bio)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001680{
1681 struct mapped_device *md = q->queuedata;
1682
1683 if (dm_request_based(md))
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001684 blk_queue_bio(q, bio);
1685 else
1686 _dm_request(q, bio);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001687}
1688
1689void dm_dispatch_request(struct request *rq)
1690{
1691 int r;
1692
1693 if (blk_queue_io_stat(rq->q))
1694 rq->cmd_flags |= REQ_IO_STAT;
1695
1696 rq->start_time = jiffies;
1697 r = blk_insert_cloned_request(rq->q, rq);
1698 if (r)
1699 dm_complete_request(rq, r);
1700}
1701EXPORT_SYMBOL_GPL(dm_dispatch_request);
1702
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001703static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
1704 void *data)
1705{
1706 struct dm_rq_target_io *tio = data;
Kent Overstreet94818742012-09-07 13:44:01 -07001707 struct dm_rq_clone_bio_info *info =
1708 container_of(bio, struct dm_rq_clone_bio_info, clone);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001709
1710 info->orig = bio_orig;
1711 info->tio = tio;
1712 bio->bi_end_io = end_clone_bio;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001713
1714 return 0;
1715}
1716
1717static int setup_clone(struct request *clone, struct request *rq,
1718 struct dm_rq_target_io *tio)
1719{
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00001720 int r;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001721
Tejun Heo29e40132010-09-08 18:07:00 +02001722 r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
1723 dm_rq_bio_constructor, tio);
1724 if (r)
1725 return r;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001726
Tejun Heo29e40132010-09-08 18:07:00 +02001727 clone->cmd = rq->cmd;
1728 clone->cmd_len = rq->cmd_len;
1729 clone->sense = rq->sense;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001730 clone->end_io = end_clone_request;
1731 clone->end_io_data = tio;
1732
1733 return 0;
1734}
1735
Kiyoshi Ueda6facdaf2009-12-10 23:52:15 +00001736static struct request *clone_rq(struct request *rq, struct mapped_device *md,
1737 gfp_t gfp_mask)
1738{
1739 struct request *clone;
1740 struct dm_rq_target_io *tio;
1741
1742 tio = alloc_rq_tio(md, gfp_mask);
1743 if (!tio)
1744 return NULL;
1745
1746 tio->md = md;
1747 tio->ti = NULL;
1748 tio->orig = rq;
1749 tio->error = 0;
1750 memset(&tio->info, 0, sizeof(tio->info));
1751
1752 clone = &tio->clone;
1753 if (setup_clone(clone, rq, tio)) {
1754 /* -ENOMEM */
1755 free_rq_tio(tio);
1756 return NULL;
1757 }
1758
1759 return clone;
1760}
1761
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001762/*
1763 * Called with the queue lock held.
1764 */
1765static int dm_prep_fn(struct request_queue *q, struct request *rq)
1766{
1767 struct mapped_device *md = q->queuedata;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001768 struct request *clone;
1769
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001770 if (unlikely(rq->special)) {
1771 DMWARN("Already has something in rq->special.");
1772 return BLKPREP_KILL;
1773 }
1774
Kiyoshi Ueda6facdaf2009-12-10 23:52:15 +00001775 clone = clone_rq(rq, md, GFP_ATOMIC);
1776 if (!clone)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001777 return BLKPREP_DEFER;
1778
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001779 rq->special = clone;
1780 rq->cmd_flags |= REQ_DONTPREP;
1781
1782 return BLKPREP_OK;
1783}
1784
Kiyoshi Ueda9eef87d2010-02-16 18:43:01 +00001785/*
1786 * Returns:
1787 * 0 : the request has been processed (not requeued)
1788 * !0 : the request has been requeued
1789 */
1790static int map_request(struct dm_target *ti, struct request *clone,
1791 struct mapped_device *md)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001792{
Kiyoshi Ueda9eef87d2010-02-16 18:43:01 +00001793 int r, requeued = 0;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001794 struct dm_rq_target_io *tio = clone->end_io_data;
1795
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001796 tio->ti = ti;
1797 r = ti->type->map_rq(ti, clone, &tio->info);
1798 switch (r) {
1799 case DM_MAPIO_SUBMITTED:
1800 /* The target has taken the I/O to submit by itself later */
1801 break;
1802 case DM_MAPIO_REMAPPED:
1803 /* The target has remapped the I/O so dispatch it */
Jun'ichi Nomura6db4ccd2009-12-10 23:52:25 +00001804 trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
1805 blk_rq_pos(tio->orig));
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001806 dm_dispatch_request(clone);
1807 break;
1808 case DM_MAPIO_REQUEUE:
1809 /* The target wants to requeue the I/O */
1810 dm_requeue_unmapped_request(clone);
Kiyoshi Ueda9eef87d2010-02-16 18:43:01 +00001811 requeued = 1;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001812 break;
1813 default:
1814 if (r > 0) {
1815 DMWARN("unimplemented target map return value: %d", r);
1816 BUG();
1817 }
1818
1819 /* The target wants to complete the I/O */
1820 dm_kill_unmapped_request(clone, r);
1821 break;
1822 }
Kiyoshi Ueda9eef87d2010-02-16 18:43:01 +00001823
1824 return requeued;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001825}
1826
Mike Snitzerba1cbad2012-09-26 23:45:42 +01001827static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
1828{
1829 struct request *clone;
1830
1831 blk_start_request(orig);
1832 clone = orig->special;
1833 atomic_inc(&md->pending[rq_data_dir(clone)]);
1834
1835 /*
1836 * Hold the md reference here for the in-flight I/O.
1837 * We can't rely on the reference count by device opener,
1838 * because the device may be closed during the request completion
1839 * when all bios are completed.
1840 * See the comment in rq_completed() too.
1841 */
1842 dm_get(md);
1843
1844 return clone;
1845}
1846
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001847/*
1848 * q->request_fn for request-based dm.
1849 * Called with the queue lock held.
1850 */
1851static void dm_request_fn(struct request_queue *q)
1852{
1853 struct mapped_device *md = q->queuedata;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001854 int srcu_idx;
1855 struct dm_table *map = dm_get_live_table(md, &srcu_idx);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001856 struct dm_target *ti;
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00001857 struct request *rq, *clone;
Tejun Heo29e40132010-09-08 18:07:00 +02001858 sector_t pos;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001859
1860 /*
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00001861 * For suspend, check blk_queue_stopped() and increment
1862 * ->pending within a single queue_lock not to increment the
1863 * number of in-flight I/Os after the queue is stopped in
1864 * dm_suspend().
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001865 */
Jens Axboe7eaceac2011-03-10 08:52:07 +01001866 while (!blk_queue_stopped(q)) {
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001867 rq = blk_peek_request(q);
1868 if (!rq)
Jens Axboe7eaceac2011-03-10 08:52:07 +01001869 goto delay_and_out;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001870
Tejun Heo29e40132010-09-08 18:07:00 +02001871 /* always use block 0 to find the target for flushes for now */
1872 pos = 0;
1873 if (!(rq->cmd_flags & REQ_FLUSH))
1874 pos = blk_rq_pos(rq);
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00001875
Tejun Heo29e40132010-09-08 18:07:00 +02001876 ti = dm_table_find_target(map, pos);
Mike Snitzerba1cbad2012-09-26 23:45:42 +01001877 if (!dm_target_is_valid(ti)) {
1878 /*
1879 * Must perform setup, that dm_done() requires,
1880 * before calling dm_kill_unmapped_request
1881 */
1882 DMERR_LIMIT("request attempted access beyond the end of device");
1883 clone = dm_start_request(md, rq);
1884 dm_kill_unmapped_request(clone, -EIO);
1885 continue;
1886 }
Tejun Heo29e40132010-09-08 18:07:00 +02001887
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001888 if (ti->type->busy && ti->type->busy(ti))
Jens Axboe7eaceac2011-03-10 08:52:07 +01001889 goto delay_and_out;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001890
Mike Snitzerba1cbad2012-09-26 23:45:42 +01001891 clone = dm_start_request(md, rq);
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00001892
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001893 spin_unlock(q->queue_lock);
Kiyoshi Ueda9eef87d2010-02-16 18:43:01 +00001894 if (map_request(ti, clone, md))
1895 goto requeued;
1896
Kiyoshi Ueda052189a2011-01-13 20:00:00 +00001897 BUG_ON(!irqs_disabled());
1898 spin_lock(q->queue_lock);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001899 }
1900
1901 goto out;
1902
Kiyoshi Ueda9eef87d2010-02-16 18:43:01 +00001903requeued:
Kiyoshi Ueda052189a2011-01-13 20:00:00 +00001904 BUG_ON(!irqs_disabled());
1905 spin_lock(q->queue_lock);
Kiyoshi Ueda9eef87d2010-02-16 18:43:01 +00001906
Jens Axboe7eaceac2011-03-10 08:52:07 +01001907delay_and_out:
1908 blk_delay_queue(q, HZ / 10);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001909out:
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001910 dm_put_live_table(md, srcu_idx);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001911}
1912
1913int dm_underlying_device_busy(struct request_queue *q)
1914{
1915 return blk_lld_busy(q);
1916}
1917EXPORT_SYMBOL_GPL(dm_underlying_device_busy);
1918
1919static int dm_lld_busy(struct request_queue *q)
1920{
1921 int r;
1922 struct mapped_device *md = q->queuedata;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001923 struct dm_table *map = dm_get_live_table_fast(md);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001924
1925 if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
1926 r = 1;
1927 else
1928 r = dm_table_any_busy_target(map);
1929
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001930 dm_put_live_table_fast(md);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001931
1932 return r;
1933}
1934
Linus Torvalds1da177e2005-04-16 15:20:36 -07001935static int dm_any_congested(void *congested_data, int bdi_bits)
1936{
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001937 int r = bdi_bits;
1938 struct mapped_device *md = congested_data;
1939 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001941 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001942 map = dm_get_live_table_fast(md);
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001943 if (map) {
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001944 /*
1945 * Request-based dm cares about only own queue for
1946 * the query about congestion status of request_queue
1947 */
1948 if (dm_request_based(md))
1949 r = md->queue->backing_dev_info.state &
1950 bdi_bits;
1951 else
1952 r = dm_table_any_congested(map, bdi_bits);
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001953 }
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001954 dm_put_live_table_fast(md);
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001955 }
1956
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 return r;
1958}
1959
1960/*-----------------------------------------------------------------
1961 * An IDR is used to keep track of allocated minor numbers.
1962 *---------------------------------------------------------------*/
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001963static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001965 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001966 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001967 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968}
1969
1970/*
1971 * See if the device with a specific minor # is free.
1972 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001973static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974{
Tejun Heoc9d76be2013-02-27 17:04:26 -08001975 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001976
1977 if (minor >= (1 << MINORBITS))
1978 return -EINVAL;
1979
Tejun Heoc9d76be2013-02-27 17:04:26 -08001980 idr_preload(GFP_KERNEL);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001981 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001982
Tejun Heoc9d76be2013-02-27 17:04:26 -08001983 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001985 spin_unlock(&_minor_lock);
Tejun Heoc9d76be2013-02-27 17:04:26 -08001986 idr_preload_end();
1987 if (r < 0)
1988 return r == -ENOSPC ? -EBUSY : r;
1989 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001990}
1991
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001992static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001993{
Tejun Heoc9d76be2013-02-27 17:04:26 -08001994 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001995
Tejun Heoc9d76be2013-02-27 17:04:26 -08001996 idr_preload(GFP_KERNEL);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001997 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998
Tejun Heoc9d76be2013-02-27 17:04:26 -08001999 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002000
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002001 spin_unlock(&_minor_lock);
Tejun Heoc9d76be2013-02-27 17:04:26 -08002002 idr_preload_end();
2003 if (r < 0)
2004 return r;
2005 *minor = r;
2006 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002007}
2008
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07002009static const struct block_device_operations dm_blk_dops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002010
Mikulas Patocka53d59142009-04-02 19:55:37 +01002011static void dm_wq_work(struct work_struct *work);
2012
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002013static void dm_init_md_queue(struct mapped_device *md)
2014{
2015 /*
2016 * Request-based dm devices cannot be stacked on top of bio-based dm
2017 * devices. The type of this dm device has not been decided yet.
2018 * The type is decided at the first table loading time.
2019 * To prevent problematic device stacking, clear the queue flag
2020 * for request stacking support until then.
2021 *
2022 * This queue is new, so no concurrency on the queue_flags.
2023 */
2024 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
2025
2026 md->queue->queuedata = md;
2027 md->queue->backing_dev_info.congested_fn = dm_any_congested;
2028 md->queue->backing_dev_info.congested_data = md;
2029 blk_queue_make_request(md->queue, dm_request);
2030 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002031 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
2032}
2033
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034/*
2035 * Allocate and initialise a blank device with a given minor.
2036 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07002037static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002038{
2039 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01002040 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07002041 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042
2043 if (!md) {
2044 DMWARN("unable to allocate device, out of memory.");
2045 return NULL;
2046 }
2047
Jeff Mahoney10da4f72006-06-26 00:27:25 -07002048 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00002049 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07002050
Linus Torvalds1da177e2005-04-16 15:20:36 -07002051 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07002052 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01002053 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07002054 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01002055 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00002057 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002059 r = init_srcu_struct(&md->io_barrier);
2060 if (r < 0)
2061 goto bad_io_barrier;
2062
Mike Snitzera5664da2010-08-12 04:14:01 +01002063 md->type = DM_TYPE_NONE;
Daniel Walkere61290a2008-02-08 02:10:08 +00002064 mutex_init(&md->suspend_lock);
Mike Snitzera5664da2010-08-12 04:14:01 +01002065 mutex_init(&md->type_lock);
Benjamin Marzinski86f11522014-08-13 13:53:43 -05002066 mutex_init(&md->table_devices_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002067 spin_lock_init(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002068 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07002069 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002071 atomic_set(&md->uevent_seq, 0);
2072 INIT_LIST_HEAD(&md->uevent_list);
Benjamin Marzinski86f11522014-08-13 13:53:43 -05002073 INIT_LIST_HEAD(&md->table_devices);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002074 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002076 md->queue = blk_alloc_queue(GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00002078 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002079
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002080 dm_init_md_queue(md);
Stefan Bader9faf4002006-10-03 01:15:41 -07002081
Linus Torvalds1da177e2005-04-16 15:20:36 -07002082 md->disk = alloc_disk(1);
2083 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00002084 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002085
Nikanth Karthikesan316d3152009-10-06 20:16:55 +02002086 atomic_set(&md->pending[0], 0);
2087 atomic_set(&md->pending[1], 0);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07002088 init_waitqueue_head(&md->wait);
Mikulas Patocka53d59142009-04-02 19:55:37 +01002089 INIT_WORK(&md->work, dm_wq_work);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07002090 init_waitqueue_head(&md->eventq);
Mikulas Patocka2995fa72014-01-13 19:37:54 -05002091 init_completion(&md->kobj_holder.completion);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07002092
Linus Torvalds1da177e2005-04-16 15:20:36 -07002093 md->disk->major = _major;
2094 md->disk->first_minor = minor;
2095 md->disk->fops = &dm_blk_dops;
2096 md->disk->queue = md->queue;
2097 md->disk->private_data = md;
2098 sprintf(md->disk->disk_name, "dm-%d", minor);
2099 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08002100 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002101
Tejun Heo670368a2013-07-30 08:40:21 -04002102 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
Milan Broz304f3f62008-02-08 02:11:17 +00002103 if (!md->wq)
2104 goto bad_thread;
2105
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002106 md->bdev = bdget_disk(md->disk, 0);
2107 if (!md->bdev)
2108 goto bad_bdev;
2109
Tejun Heo6a8736d2010-09-08 18:07:00 +02002110 bio_init(&md->flush_bio);
2111 md->flush_bio.bi_bdev = md->bdev;
2112 md->flush_bio.bi_rw = WRITE_FLUSH;
2113
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002114 dm_stats_init(&md->stats);
2115
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07002116 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002117 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07002118 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002119 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07002120
2121 BUG_ON(old_md != MINOR_ALLOCED);
2122
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123 return md;
2124
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002125bad_bdev:
2126 destroy_workqueue(md->wq);
Milan Broz304f3f62008-02-08 02:11:17 +00002127bad_thread:
Zdenek Kabelac03022c52009-10-16 23:18:15 +01002128 del_gendisk(md->disk);
Milan Broz304f3f62008-02-08 02:11:17 +00002129 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00002130bad_disk:
Al Viro1312f402006-03-12 11:02:03 -05002131 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00002132bad_queue:
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002133 cleanup_srcu_struct(&md->io_barrier);
2134bad_io_barrier:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002135 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00002136bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07002137 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00002138bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002139 kfree(md);
2140 return NULL;
2141}
2142
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01002143static void unlock_fs(struct mapped_device *md);
2144
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145static void free_dev(struct mapped_device *md)
2146{
Tejun Heof331c022008-09-03 09:01:48 +02002147 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08002148
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002149 unlock_fs(md);
2150 bdput(md->bdev);
Milan Broz304f3f62008-02-08 02:11:17 +00002151 destroy_workqueue(md->wq);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002152 if (md->io_pool)
2153 mempool_destroy(md->io_pool);
2154 if (md->bs)
2155 bioset_free(md->bs);
Martin K. Petersen9c470082009-04-09 00:27:12 +01002156 blk_integrity_unregister(md->disk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002157 del_gendisk(md->disk);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002158 cleanup_srcu_struct(&md->io_barrier);
Benjamin Marzinski86f11522014-08-13 13:53:43 -05002159 free_table_devices(&md->table_devices);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08002160 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002161
2162 spin_lock(&_minor_lock);
2163 md->disk->private_data = NULL;
2164 spin_unlock(&_minor_lock);
2165
Linus Torvalds1da177e2005-04-16 15:20:36 -07002166 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05002167 blk_cleanup_queue(md->queue);
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002168 dm_stats_cleanup(&md->stats);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07002169 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002170 kfree(md);
2171}
2172
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002173static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
2174{
Mikulas Patockac0820cf2012-12-21 20:23:38 +00002175 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002176
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00002177 if (md->io_pool && md->bs) {
Jun'ichi Nomura16245bd2013-03-01 22:45:44 +00002178 /* The md already has necessary mempools. */
2179 if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) {
2180 /*
2181 * Reload bioset because front_pad may have changed
2182 * because a different table was loaded.
2183 */
2184 bioset_free(md->bs);
2185 md->bs = p->bs;
2186 p->bs = NULL;
2187 } else if (dm_table_get_type(t) == DM_TYPE_REQUEST_BASED) {
Jun'ichi Nomura16245bd2013-03-01 22:45:44 +00002188 /*
2189 * There's no need to reload with request-based dm
2190 * because the size of front_pad doesn't change.
2191 * Note for future: If you are to reload bioset,
2192 * prep-ed requests in the queue may refer
2193 * to bio from the old bioset, so you must walk
2194 * through the queue to unprep.
2195 */
2196 }
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002197 goto out;
Mikulas Patockac0820cf2012-12-21 20:23:38 +00002198 }
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002199
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00002200 BUG_ON(!p || md->io_pool || md->bs);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002201
2202 md->io_pool = p->io_pool;
2203 p->io_pool = NULL;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002204 md->bs = p->bs;
2205 p->bs = NULL;
2206
2207out:
2208 /* mempool bind completed, now no need any mempools in the table */
2209 dm_table_free_md_mempools(t);
2210}
2211
Linus Torvalds1da177e2005-04-16 15:20:36 -07002212/*
2213 * Bind a table to the device.
2214 */
2215static void event_callback(void *context)
2216{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002217 unsigned long flags;
2218 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002219 struct mapped_device *md = (struct mapped_device *) context;
2220
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002221 spin_lock_irqsave(&md->uevent_lock, flags);
2222 list_splice_init(&md->uevent_list, &uevents);
2223 spin_unlock_irqrestore(&md->uevent_lock, flags);
2224
Tejun Heoed9e1982008-08-25 19:56:05 +09002225 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002226
Linus Torvalds1da177e2005-04-16 15:20:36 -07002227 atomic_inc(&md->event_nr);
2228 wake_up(&md->eventq);
2229}
2230
Mike Snitzerc2176492011-01-13 19:53:46 +00002231/*
2232 * Protected by md->suspend_lock obtained by dm_swap_table().
2233 */
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07002234static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002235{
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07002236 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002237
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002238 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002239}
2240
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002241/*
Mikulas Patockad5b9dd02011-08-02 12:32:04 +01002242 * Return 1 if the queue has a compulsory merge_bvec_fn function.
2243 *
2244 * If this function returns 0, then the device is either a non-dm
2245 * device without a merge_bvec_fn, or it is a dm device that is
2246 * able to split any bios it receives that are too big.
2247 */
2248int dm_queue_merge_is_compulsory(struct request_queue *q)
2249{
2250 struct mapped_device *dev_md;
2251
2252 if (!q->merge_bvec_fn)
2253 return 0;
2254
2255 if (q->make_request_fn == dm_request) {
2256 dev_md = q->queuedata;
2257 if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags))
2258 return 0;
2259 }
2260
2261 return 1;
2262}
2263
2264static int dm_device_merge_is_compulsory(struct dm_target *ti,
2265 struct dm_dev *dev, sector_t start,
2266 sector_t len, void *data)
2267{
2268 struct block_device *bdev = dev->bdev;
2269 struct request_queue *q = bdev_get_queue(bdev);
2270
2271 return dm_queue_merge_is_compulsory(q);
2272}
2273
2274/*
2275 * Return 1 if it is acceptable to ignore merge_bvec_fn based
2276 * on the properties of the underlying devices.
2277 */
2278static int dm_table_merge_is_optional(struct dm_table *table)
2279{
2280 unsigned i = 0;
2281 struct dm_target *ti;
2282
2283 while (i < dm_table_get_num_targets(table)) {
2284 ti = dm_table_get_target(table, i++);
2285
2286 if (ti->type->iterate_devices &&
2287 ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL))
2288 return 0;
2289 }
2290
2291 return 1;
2292}
2293
2294/*
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002295 * Returns old map, which caller must destroy.
2296 */
2297static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2298 struct queue_limits *limits)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002299{
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002300 struct dm_table *old_map;
Jens Axboe165125e2007-07-24 09:28:11 +02002301 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302 sector_t size;
Mikulas Patockad5b9dd02011-08-02 12:32:04 +01002303 int merge_is_optional;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002304
2305 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08002306
2307 /*
2308 * Wipe any geometry if the size of the table changed.
2309 */
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002310 if (size != dm_get_size(md))
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08002311 memset(&md->geometry, 0, sizeof(md->geometry));
2312
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002313 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002314
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002315 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002316
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002317 /*
2318 * The queue hasn't been stopped yet, if the old table type wasn't
2319 * for request-based during suspension. So stop it to prevent
2320 * I/O mapping before resume.
2321 * This must be done before setting the queue restrictions,
2322 * because request-based dm may be run just after the setting.
2323 */
2324 if (dm_table_request_based(t) && !blk_queue_stopped(q))
2325 stop_queue(q);
2326
2327 __bind_mempools(md, t);
2328
Mikulas Patockad5b9dd02011-08-02 12:32:04 +01002329 merge_is_optional = dm_table_merge_is_optional(t);
2330
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002331 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002332 rcu_assign_pointer(md->map, t);
Alasdair G Kergon36a04562011-10-31 20:19:04 +00002333 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2334
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002335 dm_table_set_restrictions(t, q, limits);
Mikulas Patockad5b9dd02011-08-02 12:32:04 +01002336 if (merge_is_optional)
2337 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2338 else
2339 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002340 if (old_map)
2341 dm_sync_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002342
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002343 return old_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002344}
2345
Alasdair G Kergona7940152009-12-10 23:52:23 +00002346/*
2347 * Returns unbound table for the caller to free.
2348 */
2349static struct dm_table *__unbind(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002350{
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002351 struct dm_table *map = rcu_dereference_protected(md->map, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352
2353 if (!map)
Alasdair G Kergona7940152009-12-10 23:52:23 +00002354 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002355
2356 dm_table_event_callback(map, NULL, NULL);
Monam Agarwal9cdb8522014-03-23 23:58:27 +05302357 RCU_INIT_POINTER(md->map, NULL);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002358 dm_sync_table(md);
Alasdair G Kergona7940152009-12-10 23:52:23 +00002359
2360 return map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002361}
2362
2363/*
2364 * Constructor for a new device.
2365 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07002366int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002367{
2368 struct mapped_device *md;
2369
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07002370 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002371 if (!md)
2372 return -ENXIO;
2373
Milan Broz784aae72009-01-06 03:05:12 +00002374 dm_sysfs_init(md);
2375
Linus Torvalds1da177e2005-04-16 15:20:36 -07002376 *result = md;
2377 return 0;
2378}
2379
Mike Snitzera5664da2010-08-12 04:14:01 +01002380/*
2381 * Functions to manage md->type.
2382 * All are required to hold md->type_lock.
2383 */
2384void dm_lock_md_type(struct mapped_device *md)
2385{
2386 mutex_lock(&md->type_lock);
2387}
2388
2389void dm_unlock_md_type(struct mapped_device *md)
2390{
2391 mutex_unlock(&md->type_lock);
2392}
2393
2394void dm_set_md_type(struct mapped_device *md, unsigned type)
2395{
Mike Snitzer00c4fc32013-08-27 18:57:03 -04002396 BUG_ON(!mutex_is_locked(&md->type_lock));
Mike Snitzera5664da2010-08-12 04:14:01 +01002397 md->type = type;
2398}
2399
2400unsigned dm_get_md_type(struct mapped_device *md)
2401{
Mike Snitzer00c4fc32013-08-27 18:57:03 -04002402 BUG_ON(!mutex_is_locked(&md->type_lock));
Mike Snitzera5664da2010-08-12 04:14:01 +01002403 return md->type;
2404}
2405
Alasdair G Kergon36a04562011-10-31 20:19:04 +00002406struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2407{
2408 return md->immutable_target_type;
2409}
2410
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002411/*
Mike Snitzerf84cb8a2013-09-19 12:13:58 -04002412 * The queue_limits are only valid as long as you have a reference
2413 * count on 'md'.
2414 */
2415struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2416{
2417 BUG_ON(!atomic_read(&md->holders));
2418 return &md->queue->limits;
2419}
2420EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2421
2422/*
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002423 * Fully initialize a request-based queue (->elevator, ->request_fn, etc).
2424 */
2425static int dm_init_request_based_queue(struct mapped_device *md)
2426{
2427 struct request_queue *q = NULL;
2428
2429 if (md->queue->elevator)
2430 return 1;
2431
2432 /* Fully initialize the queue */
2433 q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL);
2434 if (!q)
2435 return 0;
2436
2437 md->queue = q;
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002438 dm_init_md_queue(md);
2439 blk_queue_softirq_done(md->queue, dm_softirq_done);
2440 blk_queue_prep_rq(md->queue, dm_prep_fn);
2441 blk_queue_lld_busy(md->queue, dm_lld_busy);
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002442
2443 elv_register_queue(md->queue);
2444
2445 return 1;
2446}
2447
2448/*
2449 * Setup the DM device's queue based on md's type
2450 */
2451int dm_setup_md_queue(struct mapped_device *md)
2452{
2453 if ((dm_get_md_type(md) == DM_TYPE_REQUEST_BASED) &&
2454 !dm_init_request_based_queue(md)) {
2455 DMWARN("Cannot initialize queue for request-based mapped device");
2456 return -EINVAL;
2457 }
2458
2459 return 0;
2460}
2461
David Teigland637842c2006-01-06 00:20:00 -08002462static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002463{
2464 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002465 unsigned minor = MINOR(dev);
2466
2467 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2468 return NULL;
2469
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002470 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002471
2472 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002473 if (md && (md == MINOR_ALLOCED ||
Tejun Heof331c022008-09-03 09:01:48 +02002474 (MINOR(disk_devt(dm_disk(md))) != minor) ||
Kiyoshi Uedaabdc5682010-08-12 04:13:54 +01002475 dm_deleting_md(md) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07002476 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08002477 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002478 goto out;
2479 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002480
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002481out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002482 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002483
David Teigland637842c2006-01-06 00:20:00 -08002484 return md;
2485}
2486
David Teiglandd229a952006-01-06 00:20:01 -08002487struct mapped_device *dm_get_md(dev_t dev)
2488{
2489 struct mapped_device *md = dm_find_md(dev);
2490
2491 if (md)
2492 dm_get(md);
2493
2494 return md;
2495}
Alasdair G Kergon3cf2e4b2011-10-31 20:19:06 +00002496EXPORT_SYMBOL_GPL(dm_get_md);
David Teiglandd229a952006-01-06 00:20:01 -08002497
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002498void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08002499{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002500 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501}
2502
2503void dm_set_mdptr(struct mapped_device *md, void *ptr)
2504{
2505 md->interface_ptr = ptr;
2506}
2507
2508void dm_get(struct mapped_device *md)
2509{
2510 atomic_inc(&md->holders);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002511 BUG_ON(test_bit(DMF_FREEING, &md->flags));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002512}
2513
Alasdair G Kergon72d94862006-06-26 00:27:35 -07002514const char *dm_device_name(struct mapped_device *md)
2515{
2516 return md->name;
2517}
2518EXPORT_SYMBOL_GPL(dm_device_name);
2519
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002520static void __dm_destroy(struct mapped_device *md, bool wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002521{
Mike Anderson1134e5a2006-03-27 01:17:54 -08002522 struct dm_table *map;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002523 int srcu_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002524
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002525 might_sleep();
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002526
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002527 spin_lock(&_minor_lock);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002528 map = dm_get_live_table(md, &srcu_idx);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002529 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2530 set_bit(DMF_FREEING, &md->flags);
2531 spin_unlock(&_minor_lock);
2532
2533 if (!dm_suspended_md(md)) {
2534 dm_table_presuspend_targets(map);
2535 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002536 }
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002537
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002538 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2539 dm_put_live_table(md, srcu_idx);
2540
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002541 /*
2542 * Rare, but there may be I/O requests still going to complete,
2543 * for example. Wait for all references to disappear.
2544 * No one should increment the reference count of the mapped_device,
2545 * after the mapped_device state becomes DMF_FREEING.
2546 */
2547 if (wait)
2548 while (atomic_read(&md->holders))
2549 msleep(1);
2550 else if (atomic_read(&md->holders))
2551 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2552 dm_device_name(md), atomic_read(&md->holders));
2553
2554 dm_sysfs_exit(md);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002555 dm_table_destroy(__unbind(md));
2556 free_dev(md);
2557}
2558
2559void dm_destroy(struct mapped_device *md)
2560{
2561 __dm_destroy(md, true);
2562}
2563
2564void dm_destroy_immediate(struct mapped_device *md)
2565{
2566 __dm_destroy(md, false);
2567}
2568
2569void dm_put(struct mapped_device *md)
2570{
2571 atomic_dec(&md->holders);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002572}
Edward Goggin79eb8852007-05-09 02:32:56 -07002573EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002574
Mikulas Patocka401600d2009-04-02 19:55:38 +01002575static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
Milan Broz46125c12008-02-08 02:10:30 +00002576{
2577 int r = 0;
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002578 DECLARE_WAITQUEUE(wait, current);
2579
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002580 add_wait_queue(&md->wait, &wait);
Milan Broz46125c12008-02-08 02:10:30 +00002581
2582 while (1) {
Mikulas Patocka401600d2009-04-02 19:55:38 +01002583 set_current_state(interruptible);
Milan Broz46125c12008-02-08 02:10:30 +00002584
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00002585 if (!md_in_flight(md))
Milan Broz46125c12008-02-08 02:10:30 +00002586 break;
2587
Mikulas Patocka401600d2009-04-02 19:55:38 +01002588 if (interruptible == TASK_INTERRUPTIBLE &&
2589 signal_pending(current)) {
Milan Broz46125c12008-02-08 02:10:30 +00002590 r = -EINTR;
2591 break;
2592 }
2593
2594 io_schedule();
2595 }
2596 set_current_state(TASK_RUNNING);
2597
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002598 remove_wait_queue(&md->wait, &wait);
2599
Milan Broz46125c12008-02-08 02:10:30 +00002600 return r;
2601}
2602
Linus Torvalds1da177e2005-04-16 15:20:36 -07002603/*
2604 * Process the deferred bios
2605 */
Mikulas Patockaef208582009-04-02 19:55:38 +01002606static void dm_wq_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002607{
Mikulas Patockaef208582009-04-02 19:55:38 +01002608 struct mapped_device *md = container_of(work, struct mapped_device,
2609 work);
Milan Broz6d6f10d2008-02-08 02:10:22 +00002610 struct bio *c;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002611 int srcu_idx;
2612 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002613
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002614 map = dm_get_live_table(md, &srcu_idx);
Mikulas Patockaef208582009-04-02 19:55:38 +01002615
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002616 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002617 spin_lock_irq(&md->deferred_lock);
2618 c = bio_list_pop(&md->deferred);
2619 spin_unlock_irq(&md->deferred_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002620
Tejun Heo6a8736d2010-09-08 18:07:00 +02002621 if (!c)
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002622 break;
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002623
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002624 if (dm_request_based(md))
2625 generic_make_request(c);
Tejun Heo6a8736d2010-09-08 18:07:00 +02002626 else
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002627 __split_and_process_bio(md, map, c);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002628 }
Milan Broz73d410c2008-02-08 02:10:25 +00002629
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002630 dm_put_live_table(md, srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002631}
2632
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002633static void dm_queue_flush(struct mapped_device *md)
Milan Broz304f3f62008-02-08 02:11:17 +00002634{
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002635 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01002636 smp_mb__after_atomic();
Mikulas Patocka53d59142009-04-02 19:55:37 +01002637 queue_work(md->wq, &md->work);
Milan Broz304f3f62008-02-08 02:11:17 +00002638}
2639
Linus Torvalds1da177e2005-04-16 15:20:36 -07002640/*
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002641 * Swap in a new table, returning the old one for the caller to destroy.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002642 */
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002643struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002644{
Mike Christie87eb5b22013-03-01 22:45:48 +00002645 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002646 struct queue_limits limits;
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002647 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002648
Daniel Walkere61290a2008-02-08 02:10:08 +00002649 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002650
2651 /* device must be suspended */
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +00002652 if (!dm_suspended_md(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002653 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002654
Mike Snitzer3ae70652012-09-26 23:45:45 +01002655 /*
2656 * If the new table has no data devices, retain the existing limits.
2657 * This helps multipath with queue_if_no_path if all paths disappear,
2658 * then new I/O is queued based on these limits, and then some paths
2659 * reappear.
2660 */
2661 if (dm_table_has_no_data_devices(table)) {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002662 live_map = dm_get_live_table_fast(md);
Mike Snitzer3ae70652012-09-26 23:45:45 +01002663 if (live_map)
2664 limits = md->queue->limits;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002665 dm_put_live_table_fast(md);
Mike Snitzer3ae70652012-09-26 23:45:45 +01002666 }
2667
Mike Christie87eb5b22013-03-01 22:45:48 +00002668 if (!live_map) {
2669 r = dm_calculate_queue_limits(table, &limits);
2670 if (r) {
2671 map = ERR_PTR(r);
2672 goto out;
2673 }
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002674 }
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002675
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002676 map = __bind(md, table, &limits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002678out:
Daniel Walkere61290a2008-02-08 02:10:08 +00002679 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002680 return map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002681}
2682
2683/*
2684 * Functions to lock and unlock any filesystem running on the
2685 * device.
2686 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002687static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002688{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002689 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002690
2691 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002692
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002693 md->frozen_sb = freeze_bdev(md->bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002694 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002695 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002696 md->frozen_sb = NULL;
2697 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002698 }
2699
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002700 set_bit(DMF_FROZEN, &md->flags);
2701
Linus Torvalds1da177e2005-04-16 15:20:36 -07002702 return 0;
2703}
2704
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002705static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002706{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002707 if (!test_bit(DMF_FROZEN, &md->flags))
2708 return;
2709
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002710 thaw_bdev(md->bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002711 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002712 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002713}
2714
2715/*
Mike Snitzerffcc3932014-10-28 18:34:52 -04002716 * If __dm_suspend returns 0, the device is completely quiescent
2717 * now. There is no request-processing activity. All new requests
2718 * are being added to md->deferred list.
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002719 *
Mike Snitzerffcc3932014-10-28 18:34:52 -04002720 * Caller must hold md->suspend_lock
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002721 */
Mike Snitzerffcc3932014-10-28 18:34:52 -04002722static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2723 unsigned suspend_flags, int interruptible)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002724{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002725 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2726 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2727 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002728
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002729 /*
2730 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2731 * This flag is cleared before dm_suspend returns.
2732 */
2733 if (noflush)
2734 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2735
Mike Snitzerd67ee212014-10-28 20:13:31 -04002736 /*
2737 * This gets reverted if there's an error later and the targets
2738 * provide the .presuspend_undo hook.
2739 */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002740 dm_table_presuspend_targets(map);
2741
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002742 /*
Kiyoshi Ueda9f518b22009-12-10 23:52:16 +00002743 * Flush I/O to the device.
2744 * Any I/O submitted after lock_fs() may not be flushed.
2745 * noflush takes precedence over do_lockfs.
2746 * (lock_fs() flushes I/Os and waits for them to complete.)
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002747 */
2748 if (!noflush && do_lockfs) {
2749 r = lock_fs(md);
Mike Snitzerd67ee212014-10-28 20:13:31 -04002750 if (r) {
2751 dm_table_presuspend_undo_targets(map);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002752 return r;
Mike Snitzerd67ee212014-10-28 20:13:31 -04002753 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002754 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002755
2756 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002757 * Here we must make sure that no processes are submitting requests
2758 * to target drivers i.e. no one may be executing
2759 * __split_and_process_bio. This is called from dm_request and
2760 * dm_wq_work.
2761 *
2762 * To get all processes out of __split_and_process_bio in dm_request,
2763 * we take the write lock. To prevent any process from reentering
Tejun Heo6a8736d2010-09-08 18:07:00 +02002764 * __split_and_process_bio from dm_request and quiesce the thread
2765 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2766 * flush_workqueue(md->wq).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002767 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01002768 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002769 if (map)
2770 synchronize_srcu(&md->io_barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002771
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002772 /*
Tejun Heo29e40132010-09-08 18:07:00 +02002773 * Stop md->queue before flushing md->wq in case request-based
2774 * dm defers requests to md->wq from md->queue.
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002775 */
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002776 if (dm_request_based(md))
Kiyoshi Ueda9f518b22009-12-10 23:52:16 +00002777 stop_queue(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002778
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002779 flush_workqueue(md->wq);
2780
Linus Torvalds1da177e2005-04-16 15:20:36 -07002781 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002782 * At this point no more requests are entering target request routines.
2783 * We call dm_wait_for_completion to wait for all existing requests
2784 * to finish.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002785 */
Mike Snitzerffcc3932014-10-28 18:34:52 -04002786 r = dm_wait_for_completion(md, interruptible);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002787
Milan Broz6d6f10d2008-02-08 02:10:22 +00002788 if (noflush)
Mikulas Patocka022c2612009-04-02 19:55:39 +01002789 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002790 if (map)
2791 synchronize_srcu(&md->io_barrier);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002792
Linus Torvalds1da177e2005-04-16 15:20:36 -07002793 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00002794 if (r < 0) {
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002795 dm_queue_flush(md);
Milan Broz73d410c2008-02-08 02:10:25 +00002796
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002797 if (dm_request_based(md))
Kiyoshi Ueda9f518b22009-12-10 23:52:16 +00002798 start_queue(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002799
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002800 unlock_fs(md);
Mike Snitzerd67ee212014-10-28 20:13:31 -04002801 dm_table_presuspend_undo_targets(map);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002802 /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002803 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002804
Mike Snitzerffcc3932014-10-28 18:34:52 -04002805 return r;
2806}
2807
2808/*
2809 * We need to be able to change a mapping table under a mounted
2810 * filesystem. For example we might want to move some data in
2811 * the background. Before the table can be swapped with
2812 * dm_bind_table, dm_suspend must be called to flush any in
2813 * flight bios and ensure that any further io gets deferred.
2814 */
2815/*
2816 * Suspend mechanism in request-based dm.
2817 *
2818 * 1. Flush all I/Os by lock_fs() if needed.
2819 * 2. Stop dispatching any I/O by stopping the request_queue.
2820 * 3. Wait for all in-flight I/Os to be completed or requeued.
2821 *
2822 * To abort suspend, start the request_queue.
2823 */
2824int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2825{
2826 struct dm_table *map = NULL;
2827 int r = 0;
2828
2829retry:
2830 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2831
2832 if (dm_suspended_md(md)) {
2833 r = -EINVAL;
2834 goto out_unlock;
2835 }
2836
2837 if (dm_suspended_internally_md(md)) {
2838 /* already internally suspended, wait for internal resume */
2839 mutex_unlock(&md->suspend_lock);
2840 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2841 if (r)
2842 return r;
2843 goto retry;
2844 }
2845
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002846 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002847
2848 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE);
2849 if (r)
2850 goto out_unlock;
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002851
Linus Torvalds1da177e2005-04-16 15:20:36 -07002852 set_bit(DMF_SUSPENDED, &md->flags);
2853
Kiyoshi Ueda4d4471c2009-12-10 23:52:26 +00002854 dm_table_postsuspend_targets(map);
2855
Alasdair G Kergond2874832006-11-08 17:44:43 -08002856out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00002857 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002858 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002859}
2860
Mike Snitzerffcc3932014-10-28 18:34:52 -04002861static int __dm_resume(struct mapped_device *md, struct dm_table *map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002862{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002863 if (map) {
2864 int r = dm_table_resume_targets(map);
2865 if (r)
2866 return r;
2867 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002868
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002869 dm_queue_flush(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002870
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002871 /*
2872 * Flushing deferred I/Os must be done after targets are resumed
2873 * so that mapping of targets can work correctly.
2874 * Request-based dm is queueing the deferred I/Os in its request_queue.
2875 */
2876 if (dm_request_based(md))
2877 start_queue(md->queue);
2878
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002879 unlock_fs(md);
2880
Mike Snitzerffcc3932014-10-28 18:34:52 -04002881 return 0;
2882}
2883
2884int dm_resume(struct mapped_device *md)
2885{
2886 int r = -EINVAL;
2887 struct dm_table *map = NULL;
2888
2889retry:
2890 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2891
2892 if (!dm_suspended_md(md))
2893 goto out;
2894
2895 if (dm_suspended_internally_md(md)) {
2896 /* already internally suspended, wait for internal resume */
2897 mutex_unlock(&md->suspend_lock);
2898 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2899 if (r)
2900 return r;
2901 goto retry;
2902 }
2903
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002904 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002905 if (!map || !dm_table_get_size(map))
2906 goto out;
2907
2908 r = __dm_resume(md, map);
2909 if (r)
2910 goto out;
2911
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002912 clear_bit(DMF_SUSPENDED, &md->flags);
2913
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002914 r = 0;
2915out:
Daniel Walkere61290a2008-02-08 02:10:08 +00002916 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002917
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002918 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002919}
2920
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002921/*
2922 * Internal suspend/resume works like userspace-driven suspend. It waits
2923 * until all bios finish and prevents issuing new bios to the target drivers.
2924 * It may be used only from the kernel.
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002925 */
2926
Mike Snitzerffcc3932014-10-28 18:34:52 -04002927static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2928{
2929 struct dm_table *map = NULL;
2930
2931 if (dm_suspended_internally_md(md))
2932 return; /* nested internal suspend */
2933
2934 if (dm_suspended_md(md)) {
2935 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2936 return; /* nest suspend */
2937 }
2938
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002939 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002940
2941 /*
2942 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2943 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2944 * would require changing .presuspend to return an error -- avoid this
2945 * until there is a need for more elaborate variants of internal suspend.
2946 */
2947 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE);
2948
2949 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2950
2951 dm_table_postsuspend_targets(map);
2952}
2953
2954static void __dm_internal_resume(struct mapped_device *md)
2955{
2956 if (!dm_suspended_internally_md(md))
2957 return; /* resume from nested internal suspend */
2958
2959 if (dm_suspended_md(md))
2960 goto done; /* resume from nested suspend */
2961
2962 /*
2963 * NOTE: existing callers don't need to call dm_table_resume_targets
2964 * (which may fail -- so best to avoid it for now by passing NULL map)
2965 */
2966 (void) __dm_resume(md, NULL);
2967
2968done:
2969 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2970 smp_mb__after_atomic();
2971 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2972}
2973
2974void dm_internal_suspend_noflush(struct mapped_device *md)
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002975{
2976 mutex_lock(&md->suspend_lock);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002977 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2978 mutex_unlock(&md->suspend_lock);
2979}
2980EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2981
2982void dm_internal_resume(struct mapped_device *md)
2983{
2984 mutex_lock(&md->suspend_lock);
2985 __dm_internal_resume(md);
2986 mutex_unlock(&md->suspend_lock);
2987}
2988EXPORT_SYMBOL_GPL(dm_internal_resume);
2989
2990/*
2991 * Fast variants of internal suspend/resume hold md->suspend_lock,
2992 * which prevents interaction with userspace-driven suspend.
2993 */
2994
2995void dm_internal_suspend_fast(struct mapped_device *md)
2996{
2997 mutex_lock(&md->suspend_lock);
2998 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002999 return;
3000
3001 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
3002 synchronize_srcu(&md->io_barrier);
3003 flush_workqueue(md->wq);
3004 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
3005}
3006
Mike Snitzerffcc3932014-10-28 18:34:52 -04003007void dm_internal_resume_fast(struct mapped_device *md)
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04003008{
Mike Snitzerffcc3932014-10-28 18:34:52 -04003009 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04003010 goto done;
3011
3012 dm_queue_flush(md);
3013
3014done:
3015 mutex_unlock(&md->suspend_lock);
3016}
3017
Linus Torvalds1da177e2005-04-16 15:20:36 -07003018/*-----------------------------------------------------------------
3019 * Event notification.
3020 *---------------------------------------------------------------*/
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00003021int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
Milan Broz60935eb2009-06-22 10:12:30 +01003022 unsigned cookie)
Alasdair G Kergon69267a32007-12-13 14:15:57 +00003023{
Milan Broz60935eb2009-06-22 10:12:30 +01003024 char udev_cookie[DM_COOKIE_LENGTH];
3025 char *envp[] = { udev_cookie, NULL };
3026
3027 if (!cookie)
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00003028 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
Milan Broz60935eb2009-06-22 10:12:30 +01003029 else {
3030 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
3031 DM_COOKIE_ENV_VAR_NAME, cookie);
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00003032 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
3033 action, envp);
Milan Broz60935eb2009-06-22 10:12:30 +01003034 }
Alasdair G Kergon69267a32007-12-13 14:15:57 +00003035}
3036
Mike Anderson7a8c3d32007-10-19 22:48:01 +01003037uint32_t dm_next_uevent_seq(struct mapped_device *md)
3038{
3039 return atomic_add_return(1, &md->uevent_seq);
3040}
3041
Linus Torvalds1da177e2005-04-16 15:20:36 -07003042uint32_t dm_get_event_nr(struct mapped_device *md)
3043{
3044 return atomic_read(&md->event_nr);
3045}
3046
3047int dm_wait_event(struct mapped_device *md, int event_nr)
3048{
3049 return wait_event_interruptible(md->eventq,
3050 (event_nr != atomic_read(&md->event_nr)));
3051}
3052
Mike Anderson7a8c3d32007-10-19 22:48:01 +01003053void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
3054{
3055 unsigned long flags;
3056
3057 spin_lock_irqsave(&md->uevent_lock, flags);
3058 list_add(elist, &md->uevent_list);
3059 spin_unlock_irqrestore(&md->uevent_lock, flags);
3060}
3061
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062/*
3063 * The gendisk is only valid as long as you have a reference
3064 * count on 'md'.
3065 */
3066struct gendisk *dm_disk(struct mapped_device *md)
3067{
3068 return md->disk;
3069}
3070
Milan Broz784aae72009-01-06 03:05:12 +00003071struct kobject *dm_kobject(struct mapped_device *md)
3072{
Mikulas Patocka2995fa72014-01-13 19:37:54 -05003073 return &md->kobj_holder.kobj;
Milan Broz784aae72009-01-06 03:05:12 +00003074}
3075
Milan Broz784aae72009-01-06 03:05:12 +00003076struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
3077{
3078 struct mapped_device *md;
3079
Mikulas Patocka2995fa72014-01-13 19:37:54 -05003080 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
Milan Broz784aae72009-01-06 03:05:12 +00003081
Milan Broz4d89b7b2009-06-22 10:12:11 +01003082 if (test_bit(DMF_FREEING, &md->flags) ||
Mike Anderson432a2122009-12-10 23:52:20 +00003083 dm_deleting_md(md))
Milan Broz4d89b7b2009-06-22 10:12:11 +01003084 return NULL;
3085
Milan Broz784aae72009-01-06 03:05:12 +00003086 dm_get(md);
3087 return md;
3088}
3089
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +00003090int dm_suspended_md(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003091{
3092 return test_bit(DMF_SUSPENDED, &md->flags);
3093}
3094
Mike Snitzerffcc3932014-10-28 18:34:52 -04003095int dm_suspended_internally_md(struct mapped_device *md)
3096{
3097 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3098}
3099
Mikulas Patocka2c140a22013-11-01 18:27:41 -04003100int dm_test_deferred_remove_flag(struct mapped_device *md)
3101{
3102 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
3103}
3104
Kiyoshi Ueda64dbce52009-12-10 23:52:27 +00003105int dm_suspended(struct dm_target *ti)
3106{
Kiyoshi Uedaecdb2e22010-03-06 02:29:52 +00003107 return dm_suspended_md(dm_table_get_md(ti->table));
Kiyoshi Ueda64dbce52009-12-10 23:52:27 +00003108}
3109EXPORT_SYMBOL_GPL(dm_suspended);
3110
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08003111int dm_noflush_suspending(struct dm_target *ti)
3112{
Kiyoshi Uedaecdb2e22010-03-06 02:29:52 +00003113 return __noflush_suspending(dm_table_get_md(ti->table));
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08003114}
3115EXPORT_SYMBOL_GPL(dm_noflush_suspending);
3116
Mikulas Patockac0820cf2012-12-21 20:23:38 +00003117struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size)
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003118{
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003119 struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL);
3120 struct kmem_cache *cachep;
3121 unsigned int pool_size;
3122 unsigned int front_pad;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003123
3124 if (!pools)
3125 return NULL;
3126
Jun'ichi Nomura23e50832013-03-01 22:45:48 +00003127 if (type == DM_TYPE_BIO_BASED) {
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003128 cachep = _io_cache;
Mike Snitzere8603132013-09-12 18:06:12 -04003129 pool_size = dm_get_reserved_bio_based_ios();
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003130 front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
3131 } else if (type == DM_TYPE_REQUEST_BASED) {
3132 cachep = _rq_tio_cache;
Mike Snitzerf4790822013-09-12 18:06:12 -04003133 pool_size = dm_get_reserved_rq_based_ios();
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003134 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
3135 /* per_bio_data_size is not used. See __bind_mempools(). */
3136 WARN_ON(per_bio_data_size != 0);
3137 } else
3138 goto out;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003139
Mike Snitzer6cfa5852013-09-12 18:06:11 -04003140 pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003141 if (!pools->io_pool)
3142 goto out;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003143
Junichi Nomura3d8aab22014-10-03 11:55:26 +00003144 pools->bs = bioset_create_nobvec(pool_size, front_pad);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003145 if (!pools->bs)
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003146 goto out;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003147
Martin K. Petersena91a2782011-03-17 11:11:05 +01003148 if (integrity && bioset_integrity_create(pools->bs, pool_size))
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003149 goto out;
Martin K. Petersena91a2782011-03-17 11:11:05 +01003150
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003151 return pools;
3152
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003153out:
3154 dm_free_md_mempools(pools);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003155
3156 return NULL;
3157}
3158
3159void dm_free_md_mempools(struct dm_md_mempools *pools)
3160{
3161 if (!pools)
3162 return;
3163
3164 if (pools->io_pool)
3165 mempool_destroy(pools->io_pool);
3166
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003167 if (pools->bs)
3168 bioset_free(pools->bs);
3169
3170 kfree(pools);
3171}
3172
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07003173static const struct block_device_operations dm_blk_dops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003174 .open = dm_blk_open,
3175 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07003176 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08003177 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003178 .owner = THIS_MODULE
3179};
3180
Linus Torvalds1da177e2005-04-16 15:20:36 -07003181/*
3182 * module hooks
3183 */
3184module_init(dm_init);
3185module_exit(dm_exit);
3186
3187module_param(major, uint, 0);
3188MODULE_PARM_DESC(major, "The major number of the device mapper");
Mike Snitzerf4790822013-09-12 18:06:12 -04003189
Mike Snitzere8603132013-09-12 18:06:12 -04003190module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3191MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3192
Mike Snitzerf4790822013-09-12 18:06:12 -04003193module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR);
3194MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools");
3195
Linus Torvalds1da177e2005-04-16 15:20:36 -07003196MODULE_DESCRIPTION(DM_NAME " driver");
3197MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3198MODULE_LICENSE("GPL");