blob: bc143c1b23339bd430cf72d39a8ed17449389e83 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Milan Broz784aae72009-01-06 03:05:12 +00003 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
Mike Snitzer4cc96132016-05-12 16:28:10 -04008#include "dm-core.h"
9#include "dm-rq.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +010010#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12#include <linux/init.h>
13#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080014#include <linux/mutex.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010015#include <linux/sched/signal.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/blkpg.h>
17#include <linux/bio.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/mempool.h>
Dan Williamsf26c5712017-04-12 12:35:44 -070019#include <linux/dax.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/slab.h>
21#include <linux/idr.h>
Dan Williams7e026c82017-05-29 12:57:56 -070022#include <linux/uio.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080023#include <linux/hdreg.h>
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +010024#include <linux/delay.h>
Mike Snitzerffcc3932014-10-28 18:34:52 -040025#include <linux/wait.h>
Christoph Hellwig71cdb692015-10-15 14:10:51 +020026#include <linux/pr.h>
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +030027#include <linux/refcount.h>
Li Zefan55782132009-06-09 13:43:05 +080028
Alasdair G Kergon72d94862006-06-26 00:27:35 -070029#define DM_MSG_PREFIX "core"
30
Milan Broz60935eb2009-06-22 10:12:30 +010031/*
32 * Cookies are numeric values sent with CHANGE and REMOVE
33 * uevents while resuming, removing or renaming the device.
34 */
35#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
36#define DM_COOKIE_LENGTH 24
37
Linus Torvalds1da177e2005-04-16 15:20:36 -070038static const char *_name = DM_NAME;
39
40static unsigned int major = 0;
41static unsigned int _major = 0;
42
Alasdair G Kergond15b7742011-08-02 12:32:01 +010043static DEFINE_IDR(_minor_idr);
44
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070045static DEFINE_SPINLOCK(_minor_lock);
Mikulas Patocka2c140a22013-11-01 18:27:41 -040046
47static void do_deferred_remove(struct work_struct *w);
48
49static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
50
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -040051static struct workqueue_struct *deferred_remove_workqueue;
52
Mikulas Patocka93e64422017-01-16 16:05:59 -050053atomic_t dm_global_event_nr = ATOMIC_INIT(0);
54DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
55
Mikulas Patocka62e08242017-09-20 07:29:49 -040056void dm_issue_global_event(void)
57{
58 atomic_inc(&dm_global_event_nr);
59 wake_up(&dm_global_eventq);
60}
61
Linus Torvalds1da177e2005-04-16 15:20:36 -070062/*
Mike Snitzer64f52b02017-12-11 23:17:47 -050063 * One of these is allocated (on-stack) per original bio.
Linus Torvalds1da177e2005-04-16 15:20:36 -070064 */
Mike Snitzer64f52b02017-12-11 23:17:47 -050065struct clone_info {
Mike Snitzer64f52b02017-12-11 23:17:47 -050066 struct dm_table *map;
67 struct bio *bio;
68 struct dm_io *io;
69 sector_t sector;
70 unsigned sector_count;
71};
72
73/*
74 * One of these is allocated per clone bio.
75 */
76#define DM_TIO_MAGIC 7282014
77struct dm_target_io {
78 unsigned magic;
79 struct dm_io *io;
80 struct dm_target *ti;
81 unsigned target_bio_nr;
82 unsigned *len_ptr;
83 bool inside_dm_io;
84 struct bio clone;
85};
86
87/*
88 * One of these is allocated per original bio.
89 * It contains the first clone used for that original.
90 */
91#define DM_IO_MAGIC 5191977
Linus Torvalds1da177e2005-04-16 15:20:36 -070092struct dm_io {
Mike Snitzer64f52b02017-12-11 23:17:47 -050093 unsigned magic;
Linus Torvalds1da177e2005-04-16 15:20:36 -070094 struct mapped_device *md;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +020095 blk_status_t status;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 atomic_t io_count;
Mike Snitzer745dc572017-12-11 20:51:50 -050097 struct bio *orig_bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080098 unsigned long start_time;
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +010099 spinlock_t endio_lock;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400100 struct dm_stats_aux stats_aux;
Mike Snitzer64f52b02017-12-11 23:17:47 -0500101 /* last member of dm_target_io is 'struct bio' */
102 struct dm_target_io tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103};
104
Mike Snitzer64f52b02017-12-11 23:17:47 -0500105void *dm_per_bio_data(struct bio *bio, size_t data_size)
106{
107 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
108 if (!tio->inside_dm_io)
109 return (char *)bio - offsetof(struct dm_target_io, clone) - data_size;
110 return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size;
111}
112EXPORT_SYMBOL_GPL(dm_per_bio_data);
113
114struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
115{
116 struct dm_io *io = (struct dm_io *)((char *)data + data_size);
117 if (io->magic == DM_IO_MAGIC)
118 return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone));
119 BUG_ON(io->magic != DM_TIO_MAGIC);
120 return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone));
121}
122EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
123
124unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
125{
126 return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
127}
128EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
129
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700130#define MINOR_ALLOCED ((void *)-1)
131
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132/*
133 * Bits for the md->flags field.
134 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +0100135#define DMF_BLOCK_IO_FOR_SUSPEND 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -0800137#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700138#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700139#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800140#define DMF_NOFLUSH_SUSPENDING 5
Kent Overstreet8ae12662015-04-27 23:48:34 -0700141#define DMF_DEFERRED_REMOVE 6
142#define DMF_SUSPENDED_INTERNALLY 7
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143
Mike Snitzer115485e2016-02-22 12:16:21 -0500144#define DM_NUMA_NODE NUMA_NO_NODE
Mike Snitzer115485e2016-02-22 12:16:21 -0500145static int dm_numa_node = DM_NUMA_NODE;
Mike Snitzerfaad87d2016-01-28 16:52:56 -0500146
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +0100147/*
148 * For mempools pre-allocation at the table loading time.
149 */
150struct dm_md_mempools {
Kent Overstreet6f1c8192018-05-20 18:25:53 -0400151 struct bio_set bs;
152 struct bio_set io_bs;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +0100153};
154
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500155struct table_device {
156 struct list_head list;
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300157 refcount_t count;
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500158 struct dm_dev dm_dev;
159};
160
Mike Snitzerf4790822013-09-12 18:06:12 -0400161/*
Mike Snitzere8603132013-09-12 18:06:12 -0400162 * Bio-based DM's mempools' reserved IOs set by the user.
163 */
Mike Snitzer4cc96132016-05-12 16:28:10 -0400164#define RESERVED_BIO_BASED_IOS 16
Mike Snitzere8603132013-09-12 18:06:12 -0400165static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
166
Mike Snitzer115485e2016-02-22 12:16:21 -0500167static int __dm_get_module_param_int(int *module_param, int min, int max)
168{
Mark Rutland6aa7de02017-10-23 14:07:29 -0700169 int param = READ_ONCE(*module_param);
Mike Snitzer115485e2016-02-22 12:16:21 -0500170 int modified_param = 0;
171 bool modified = true;
172
173 if (param < min)
174 modified_param = min;
175 else if (param > max)
176 modified_param = max;
177 else
178 modified = false;
179
180 if (modified) {
181 (void)cmpxchg(module_param, param, modified_param);
182 param = modified_param;
183 }
184
185 return param;
186}
187
Mike Snitzer4cc96132016-05-12 16:28:10 -0400188unsigned __dm_get_module_param(unsigned *module_param,
189 unsigned def, unsigned max)
Mike Snitzerf4790822013-09-12 18:06:12 -0400190{
Mark Rutland6aa7de02017-10-23 14:07:29 -0700191 unsigned param = READ_ONCE(*module_param);
Mike Snitzer09c2d532015-02-27 22:25:26 -0500192 unsigned modified_param = 0;
Mike Snitzerf4790822013-09-12 18:06:12 -0400193
Mike Snitzer09c2d532015-02-27 22:25:26 -0500194 if (!param)
195 modified_param = def;
196 else if (param > max)
197 modified_param = max;
Mike Snitzerf4790822013-09-12 18:06:12 -0400198
Mike Snitzer09c2d532015-02-27 22:25:26 -0500199 if (modified_param) {
200 (void)cmpxchg(module_param, param, modified_param);
201 param = modified_param;
Mike Snitzerf4790822013-09-12 18:06:12 -0400202 }
203
Mike Snitzer09c2d532015-02-27 22:25:26 -0500204 return param;
Mike Snitzerf4790822013-09-12 18:06:12 -0400205}
206
Mike Snitzere8603132013-09-12 18:06:12 -0400207unsigned dm_get_reserved_bio_based_ios(void)
208{
Mike Snitzer09c2d532015-02-27 22:25:26 -0500209 return __dm_get_module_param(&reserved_bio_based_ios,
Mike Snitzer4cc96132016-05-12 16:28:10 -0400210 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
Mike Snitzere8603132013-09-12 18:06:12 -0400211}
212EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
213
Mike Snitzer115485e2016-02-22 12:16:21 -0500214static unsigned dm_get_numa_node(void)
215{
216 return __dm_get_module_param_int(&dm_numa_node,
217 DM_NUMA_NODE, num_online_nodes() - 1);
218}
219
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220static int __init local_init(void)
221{
Mike Snitzere689fba2019-02-20 15:37:44 -0500222 int r;
Mike Snitzer1ae49ea2014-12-05 17:11:05 -0500223
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100224 r = dm_uevent_init();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100225 if (r)
Mike Snitzere689fba2019-02-20 15:37:44 -0500226 return r;
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100227
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400228 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
229 if (!deferred_remove_workqueue) {
230 r = -ENOMEM;
231 goto out_uevent_exit;
232 }
233
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 _major = major;
235 r = register_blkdev(_major, _name);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100236 if (r < 0)
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400237 goto out_free_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
239 if (!_major)
240 _major = r;
241
242 return 0;
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100243
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400244out_free_workqueue:
245 destroy_workqueue(deferred_remove_workqueue);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100246out_uevent_exit:
247 dm_uevent_exit();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100248
249 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250}
251
252static void local_exit(void)
253{
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400254 flush_scheduled_work();
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400255 destroy_workqueue(deferred_remove_workqueue);
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400256
Akinobu Mita00d59402007-07-17 04:03:46 -0700257 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100258 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
260 _major = 0;
261
262 DMINFO("cleaned up");
263}
264
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000265static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 local_init,
267 dm_target_init,
268 dm_linear_init,
269 dm_stripe_init,
Mikulas Patocka952b3552009-12-10 23:51:57 +0000270 dm_io_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100271 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 dm_interface_init,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400273 dm_statistics_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274};
275
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000276static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 local_exit,
278 dm_target_exit,
279 dm_linear_exit,
280 dm_stripe_exit,
Mikulas Patocka952b3552009-12-10 23:51:57 +0000281 dm_io_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100282 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 dm_interface_exit,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400284 dm_statistics_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285};
286
287static int __init dm_init(void)
288{
289 const int count = ARRAY_SIZE(_inits);
290
291 int r, i;
292
293 for (i = 0; i < count; i++) {
294 r = _inits[i]();
295 if (r)
296 goto bad;
297 }
298
299 return 0;
300
301 bad:
302 while (i--)
303 _exits[i]();
304
305 return r;
306}
307
308static void __exit dm_exit(void)
309{
310 int i = ARRAY_SIZE(_exits);
311
312 while (i--)
313 _exits[i]();
Alasdair G Kergond15b7742011-08-02 12:32:01 +0100314
315 /*
316 * Should be empty by this point.
317 */
Alasdair G Kergond15b7742011-08-02 12:32:01 +0100318 idr_destroy(&_minor_idr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319}
320
321/*
322 * Block device functions
323 */
Mike Anderson432a2122009-12-10 23:52:20 +0000324int dm_deleting_md(struct mapped_device *md)
325{
326 return test_bit(DMF_DELETING, &md->flags);
327}
328
Al Virofe5f9f22008-03-02 10:29:31 -0500329static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330{
331 struct mapped_device *md;
332
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700333 spin_lock(&_minor_lock);
334
Al Virofe5f9f22008-03-02 10:29:31 -0500335 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700336 if (!md)
337 goto out;
338
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700339 if (test_bit(DMF_FREEING, &md->flags) ||
Mike Anderson432a2122009-12-10 23:52:20 +0000340 dm_deleting_md(md)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700341 md = NULL;
342 goto out;
343 }
344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700346 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700347out:
348 spin_unlock(&_minor_lock);
349
350 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351}
352
Al Virodb2a1442013-05-05 21:52:57 -0400353static void dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354{
Mike Snitzer63a4f062015-03-23 17:01:43 -0400355 struct mapped_device *md;
Arnd Bergmann6e9624b2010-08-07 18:25:34 +0200356
Milan Broz4a1aeb92011-01-13 19:59:48 +0000357 spin_lock(&_minor_lock);
358
Mike Snitzer63a4f062015-03-23 17:01:43 -0400359 md = disk->private_data;
360 if (WARN_ON(!md))
361 goto out;
362
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400363 if (atomic_dec_and_test(&md->open_count) &&
364 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400365 queue_work(deferred_remove_workqueue, &deferred_remove_work);
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400366
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 dm_put(md);
Mike Snitzer63a4f062015-03-23 17:01:43 -0400368out:
Milan Broz4a1aeb92011-01-13 19:59:48 +0000369 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370}
371
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700372int dm_open_count(struct mapped_device *md)
373{
374 return atomic_read(&md->open_count);
375}
376
377/*
378 * Guarantees nothing is using the device before it's deleted.
379 */
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400380int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700381{
382 int r = 0;
383
384 spin_lock(&_minor_lock);
385
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400386 if (dm_open_count(md)) {
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700387 r = -EBUSY;
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400388 if (mark_deferred)
389 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
390 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
391 r = -EEXIST;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700392 else
393 set_bit(DMF_DELETING, &md->flags);
394
395 spin_unlock(&_minor_lock);
396
397 return r;
398}
399
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400400int dm_cancel_deferred_remove(struct mapped_device *md)
401{
402 int r = 0;
403
404 spin_lock(&_minor_lock);
405
406 if (test_bit(DMF_DELETING, &md->flags))
407 r = -EBUSY;
408 else
409 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
410
411 spin_unlock(&_minor_lock);
412
413 return r;
414}
415
416static void do_deferred_remove(struct work_struct *w)
417{
418 dm_deferred_remove();
419}
420
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400421sector_t dm_get_size(struct mapped_device *md)
422{
423 return get_capacity(md->disk);
424}
425
Mike Snitzer9974fa22014-02-28 15:33:43 +0100426struct request_queue *dm_get_md_queue(struct mapped_device *md)
427{
428 return md->queue;
429}
430
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400431struct dm_stats *dm_get_stats(struct mapped_device *md)
432{
433 return &md->stats;
434}
435
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800436static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
437{
438 struct mapped_device *md = bdev->bd_disk->private_data;
439
440 return dm_get_geometry(md, geo);
441}
442
Christoph Hellwige76239a2018-10-12 19:08:49 +0900443static int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
Damien Le Moalbd976e52019-07-01 14:09:16 +0900444 struct blk_zone *zones, unsigned int *nr_zones)
Christoph Hellwige76239a2018-10-12 19:08:49 +0900445{
446#ifdef CONFIG_BLK_DEV_ZONED
447 struct mapped_device *md = disk->private_data;
448 struct dm_target *tgt;
449 struct dm_table *map;
450 int srcu_idx, ret;
451
452 if (dm_suspended_md(md))
453 return -EAGAIN;
454
455 map = dm_get_live_table(md, &srcu_idx);
456 if (!map)
457 return -EIO;
458
459 tgt = dm_table_find_target(map, sector);
Mikulas Patocka123d87d2019-08-23 09:55:26 -0400460 if (!tgt) {
Christoph Hellwige76239a2018-10-12 19:08:49 +0900461 ret = -EIO;
462 goto out;
463 }
464
465 /*
466 * If we are executing this, we already know that the block device
467 * is a zoned device and so each target should have support for that
468 * type of drive. A missing report_zones method means that the target
469 * driver has a problem.
470 */
471 if (WARN_ON(!tgt->type->report_zones)) {
472 ret = -EIO;
473 goto out;
474 }
475
476 /*
477 * blkdev_report_zones() will loop and call this again to cover all the
478 * zones of the target, eventually moving on to the next target.
479 * So there is no need to loop here trying to fill the entire array
480 * of zones.
481 */
Damien Le Moalbd976e52019-07-01 14:09:16 +0900482 ret = tgt->type->report_zones(tgt, sector, zones, nr_zones);
Christoph Hellwige76239a2018-10-12 19:08:49 +0900483
484out:
485 dm_put_live_table(md, srcu_idx);
486 return ret;
487#else
488 return -ENOTSUPP;
489#endif
490}
491
Mike Snitzer971888c2018-04-03 15:05:12 -0400492static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
Mike Snitzer5bd5e8d2018-04-03 16:54:10 -0400493 struct block_device **bdev)
Mike Snitzer971888c2018-04-03 15:05:12 -0400494 __acquires(md->io_barrier)
Milan Brozaa129a22006-10-03 01:15:15 -0700495{
Mike Snitzer66482022016-02-18 15:44:39 -0500496 struct dm_target *tgt;
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100497 struct dm_table *map;
Mike Snitzer971888c2018-04-03 15:05:12 -0400498 int r;
Milan Brozaa129a22006-10-03 01:15:15 -0700499
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100500retry:
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200501 r = -ENOTTY;
Mike Snitzer971888c2018-04-03 15:05:12 -0400502 map = dm_get_live_table(md, srcu_idx);
Milan Brozaa129a22006-10-03 01:15:15 -0700503 if (!map || !dm_table_get_size(map))
Mike Snitzer971888c2018-04-03 15:05:12 -0400504 return r;
Milan Brozaa129a22006-10-03 01:15:15 -0700505
506 /* We only support devices that have a single target */
507 if (dm_table_get_num_targets(map) != 1)
Mike Snitzer971888c2018-04-03 15:05:12 -0400508 return r;
Milan Brozaa129a22006-10-03 01:15:15 -0700509
Mike Snitzer66482022016-02-18 15:44:39 -0500510 tgt = dm_table_get_target(map, 0);
511 if (!tgt->type->prepare_ioctl)
Mike Snitzer971888c2018-04-03 15:05:12 -0400512 return r;
Milan Brozaa129a22006-10-03 01:15:15 -0700513
Mike Snitzer971888c2018-04-03 15:05:12 -0400514 if (dm_suspended_md(md))
515 return -EAGAIN;
Milan Brozaa129a22006-10-03 01:15:15 -0700516
Mike Snitzer5bd5e8d2018-04-03 16:54:10 -0400517 r = tgt->type->prepare_ioctl(tgt, bdev);
Junichi Nomura5bbbfdf2015-11-17 09:39:26 +0000518 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
Mike Snitzer971888c2018-04-03 15:05:12 -0400519 dm_put_live_table(md, *srcu_idx);
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100520 msleep(10);
521 goto retry;
522 }
Mike Snitzer971888c2018-04-03 15:05:12 -0400523
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200524 return r;
525}
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100526
Mike Snitzer971888c2018-04-03 15:05:12 -0400527static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
528 __releases(md->io_barrier)
529{
530 dm_put_live_table(md, srcu_idx);
531}
532
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200533static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
534 unsigned int cmd, unsigned long arg)
535{
536 struct mapped_device *md = bdev->bd_disk->private_data;
Mike Snitzer971888c2018-04-03 15:05:12 -0400537 int r, srcu_idx;
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200538
Mike Snitzer5bd5e8d2018-04-03 16:54:10 -0400539 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200540 if (r < 0)
Mike Snitzer971888c2018-04-03 15:05:12 -0400541 goto out;
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200542
543 if (r > 0) {
544 /*
Christoph Hellwige980f622017-02-04 10:45:03 +0100545 * Target determined this ioctl is being issued against a
546 * subset of the parent bdev; require extra privileges.
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200547 */
Christoph Hellwige980f622017-02-04 10:45:03 +0100548 if (!capable(CAP_SYS_RAWIO)) {
549 DMWARN_LIMIT(
550 "%s: sending ioctl %x to DM device without required privilege.",
551 current->comm, cmd);
552 r = -ENOIOCTLCMD;
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200553 goto out;
Christoph Hellwige980f622017-02-04 10:45:03 +0100554 }
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200555 }
556
Mike Snitzer66482022016-02-18 15:44:39 -0500557 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg);
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200558out:
Mike Snitzer971888c2018-04-03 15:05:12 -0400559 dm_unprepare_ioctl(md, srcu_idx);
Milan Brozaa129a22006-10-03 01:15:15 -0700560 return r;
561}
562
Mike Snitzer978e51b2017-12-09 15:16:42 -0500563static void start_io_acct(struct dm_io *io);
564
565static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566{
Mike Snitzer64f52b02017-12-11 23:17:47 -0500567 struct dm_io *io;
568 struct dm_target_io *tio;
569 struct bio *clone;
570
Kent Overstreet6f1c8192018-05-20 18:25:53 -0400571 clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
Mike Snitzer64f52b02017-12-11 23:17:47 -0500572 if (!clone)
573 return NULL;
574
575 tio = container_of(clone, struct dm_target_io, clone);
576 tio->inside_dm_io = true;
577 tio->io = NULL;
578
579 io = container_of(tio, struct dm_io, tio);
580 io->magic = DM_IO_MAGIC;
Mike Snitzer978e51b2017-12-09 15:16:42 -0500581 io->status = 0;
582 atomic_set(&io->io_count, 1);
583 io->orig_bio = bio;
584 io->md = md;
585 spin_lock_init(&io->endio_lock);
586
587 start_io_acct(io);
Mike Snitzer64f52b02017-12-11 23:17:47 -0500588
589 return io;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590}
591
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100592static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593{
Mike Snitzer64f52b02017-12-11 23:17:47 -0500594 bio_put(&io->tio.clone);
595}
596
597static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
598 unsigned target_bio_nr, gfp_t gfp_mask)
599{
600 struct dm_target_io *tio;
601
602 if (!ci->io->tio.io) {
603 /* the dm_target_io embedded in ci->io is available */
604 tio = &ci->io->tio;
605 } else {
Kent Overstreet6f1c8192018-05-20 18:25:53 -0400606 struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
Mike Snitzer64f52b02017-12-11 23:17:47 -0500607 if (!clone)
608 return NULL;
609
610 tio = container_of(clone, struct dm_target_io, clone);
611 tio->inside_dm_io = false;
612 }
613
614 tio->magic = DM_TIO_MAGIC;
615 tio->io = ci->io;
616 tio->ti = ti;
617 tio->target_bio_nr = target_bio_nr;
618
619 return tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620}
621
Mike Snitzercfae7522016-04-11 12:05:38 -0400622static void free_tio(struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623{
Mike Snitzer64f52b02017-12-11 23:17:47 -0500624 if (tio->inside_dm_io)
625 return;
Mikulas Patockadba14162012-10-12 21:02:15 +0100626 bio_put(&tio->clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627}
628
Mike Snitzerc4576ae2018-12-11 09:10:26 -0500629static bool md_in_flight_bios(struct mapped_device *md)
Kiyoshi Ueda90abb8c2009-12-10 23:52:13 +0000630{
Mikulas Patocka6f757232018-12-06 11:41:22 -0500631 int cpu;
632 struct hd_struct *part = &dm_disk(md)->part0;
Jens Axboeb7934ba2018-12-10 15:45:53 -0700633 long sum = 0;
Mikulas Patocka6f757232018-12-06 11:41:22 -0500634
635 for_each_possible_cpu(cpu) {
Jens Axboeb7934ba2018-12-10 15:45:53 -0700636 sum += part_stat_local_read_cpu(part, in_flight[0], cpu);
637 sum += part_stat_local_read_cpu(part, in_flight[1], cpu);
Mikulas Patocka6f757232018-12-06 11:41:22 -0500638 }
639
Jens Axboeb7934ba2018-12-10 15:45:53 -0700640 return sum != 0;
Kiyoshi Ueda90abb8c2009-12-10 23:52:13 +0000641}
642
Mike Snitzerc4576ae2018-12-11 09:10:26 -0500643static bool md_in_flight(struct mapped_device *md)
644{
645 if (queue_is_mq(md->queue))
Jens Axboe3c94d832018-12-17 21:11:17 -0700646 return blk_mq_queue_inflight(md->queue);
Mike Snitzerc4576ae2018-12-11 09:10:26 -0500647 else
648 return md_in_flight_bios(md);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100649}
650
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800651static void start_io_acct(struct dm_io *io)
652{
653 struct mapped_device *md = io->md;
Mike Snitzer745dc572017-12-11 20:51:50 -0500654 struct bio *bio = io->orig_bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800655
656 io->start_time = jiffies;
657
Michael Callahanddcf35d2018-07-18 04:47:39 -0700658 generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
659 &dm_disk(md)->part0);
Mike Snitzerf3986372017-12-17 11:56:48 -0500660
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400661 if (unlikely(dm_stats_used(&md->stats)))
Mike Christie528ec5a2016-06-05 14:32:03 -0500662 dm_stats_account_io(&md->stats, bio_data_dir(bio),
663 bio->bi_iter.bi_sector, bio_sectors(bio),
664 false, 0, &io->stats_aux);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800665}
666
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000667static void end_io_acct(struct dm_io *io)
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800668{
669 struct mapped_device *md = io->md;
Mike Snitzer745dc572017-12-11 20:51:50 -0500670 struct bio *bio = io->orig_bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800671 unsigned long duration = jiffies - io->start_time;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800672
Michael Callahanddcf35d2018-07-18 04:47:39 -0700673 generic_end_io_acct(md->queue, bio_op(bio), &dm_disk(md)->part0,
674 io->start_time);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800675
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400676 if (unlikely(dm_stats_used(&md->stats)))
Mike Christie528ec5a2016-06-05 14:32:03 -0500677 dm_stats_account_io(&md->stats, bio_data_dir(bio),
678 bio->bi_iter.bi_sector, bio_sectors(bio),
679 true, duration, &io->stats_aux);
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400680
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000681 /* nudge anyone waiting on suspend queue */
Mikulas Patocka645efa82019-02-05 05:09:00 -0500682 if (unlikely(wq_has_sleeper(&md->wait)))
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000683 wake_up(&md->wait);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800684}
685
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686/*
687 * Add the bio to the list of deferred io.
688 */
Mikulas Patocka92c63902009-04-09 00:27:15 +0100689static void queue_io(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690{
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200691 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200693 spin_lock_irqsave(&md->deferred_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 bio_list_add(&md->deferred, bio);
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200695 spin_unlock_irqrestore(&md->deferred_lock, flags);
Tejun Heo6a8736d2010-09-08 18:07:00 +0200696 queue_work(md->wq, &md->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697}
698
699/*
700 * Everyone (including functions in this file), should use this
701 * function to access the md->map field, and make sure they call
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100702 * dm_put_live_table() when finished.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 */
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100704struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705{
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100706 *srcu_idx = srcu_read_lock(&md->io_barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100708 return srcu_dereference(md->map, &md->io_barrier);
709}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100711void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
712{
713 srcu_read_unlock(&md->io_barrier, srcu_idx);
714}
715
716void dm_sync_table(struct mapped_device *md)
717{
718 synchronize_srcu(&md->io_barrier);
719 synchronize_rcu_expedited();
720}
721
722/*
723 * A fast alternative to dm_get_live_table/dm_put_live_table.
724 * The caller must not block between these two functions.
725 */
726static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
727{
728 rcu_read_lock();
729 return rcu_dereference(md->map);
730}
731
732static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
733{
734 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735}
736
Mike Snitzer971888c2018-04-03 15:05:12 -0400737static char *_dm_claim_ptr = "I belong to device-mapper";
738
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800739/*
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500740 * Open a table device so we can use it as a map destination.
741 */
742static int open_table_device(struct table_device *td, dev_t dev,
743 struct mapped_device *md)
744{
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500745 struct block_device *bdev;
746
747 int r;
748
749 BUG_ON(td->dm_dev.bdev);
750
Mike Snitzer519049a2018-02-22 13:31:20 -0500751 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500752 if (IS_ERR(bdev))
753 return PTR_ERR(bdev);
754
755 r = bd_link_disk_holder(bdev, dm_disk(md));
756 if (r) {
757 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
758 return r;
759 }
760
761 td->dm_dev.bdev = bdev;
Dan Williams817bf402017-04-12 13:37:44 -0700762 td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500763 return 0;
764}
765
766/*
767 * Close a table device that we've been using.
768 */
769static void close_table_device(struct table_device *td, struct mapped_device *md)
770{
771 if (!td->dm_dev.bdev)
772 return;
773
774 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
775 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
Dan Williams817bf402017-04-12 13:37:44 -0700776 put_dax(td->dm_dev.dax_dev);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500777 td->dm_dev.bdev = NULL;
Dan Williams817bf402017-04-12 13:37:44 -0700778 td->dm_dev.dax_dev = NULL;
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500779}
780
781static struct table_device *find_table_device(struct list_head *l, dev_t dev,
Sheetal Singala8454fca2019-05-10 23:18:37 +0530782 fmode_t mode)
783{
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500784 struct table_device *td;
785
786 list_for_each_entry(td, l, list)
787 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
788 return td;
789
790 return NULL;
791}
792
793int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
Sheetal Singala8454fca2019-05-10 23:18:37 +0530794 struct dm_dev **result)
795{
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500796 int r;
797 struct table_device *td;
798
799 mutex_lock(&md->table_devices_lock);
800 td = find_table_device(&md->table_devices, dev, mode);
801 if (!td) {
Mike Snitzer115485e2016-02-22 12:16:21 -0500802 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500803 if (!td) {
804 mutex_unlock(&md->table_devices_lock);
805 return -ENOMEM;
806 }
807
808 td->dm_dev.mode = mode;
809 td->dm_dev.bdev = NULL;
810
811 if ((r = open_table_device(td, dev, md))) {
812 mutex_unlock(&md->table_devices_lock);
813 kfree(td);
814 return r;
815 }
816
817 format_dev_t(td->dm_dev.name, dev);
818
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300819 refcount_set(&td->count, 1);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500820 list_add(&td->list, &md->table_devices);
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300821 } else {
822 refcount_inc(&td->count);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500823 }
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500824 mutex_unlock(&md->table_devices_lock);
825
826 *result = &td->dm_dev;
827 return 0;
828}
829EXPORT_SYMBOL_GPL(dm_get_table_device);
830
831void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
832{
833 struct table_device *td = container_of(d, struct table_device, dm_dev);
834
835 mutex_lock(&md->table_devices_lock);
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300836 if (refcount_dec_and_test(&td->count)) {
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500837 close_table_device(td, md);
838 list_del(&td->list);
839 kfree(td);
840 }
841 mutex_unlock(&md->table_devices_lock);
842}
843EXPORT_SYMBOL(dm_put_table_device);
844
845static void free_table_devices(struct list_head *devices)
846{
847 struct list_head *tmp, *next;
848
849 list_for_each_safe(tmp, next, devices) {
850 struct table_device *td = list_entry(tmp, struct table_device, list);
851
852 DMWARN("dm_destroy: %s still exists with %d references",
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300853 td->dm_dev.name, refcount_read(&td->count));
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500854 kfree(td);
855 }
856}
857
858/*
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800859 * Get the geometry associated with a dm device
860 */
861int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
862{
863 *geo = md->geometry;
864
865 return 0;
866}
867
868/*
869 * Set the geometry of a device.
870 */
871int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
872{
873 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
874
875 if (geo->start > sz) {
876 DMWARN("Start sector is beyond the geometry limits.");
877 return -EINVAL;
878 }
879
880 md->geometry = *geo;
881
882 return 0;
883}
884
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800885static int __noflush_suspending(struct mapped_device *md)
886{
887 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
888}
889
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890/*
891 * Decrements the number of outstanding ios that a bio has been
892 * cloned into, completing the original io if necc.
893 */
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200894static void dec_pending(struct dm_io *io, blk_status_t error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800896 unsigned long flags;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200897 blk_status_t io_error;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000898 struct bio *bio;
899 struct mapped_device *md = io->md;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800900
901 /* Push-back supersedes any I/O errors */
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +0100902 if (unlikely(error)) {
903 spin_lock_irqsave(&io->endio_lock, flags);
Mike Snitzer745dc572017-12-11 20:51:50 -0500904 if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200905 io->status = error;
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +0100906 spin_unlock_irqrestore(&io->endio_lock, flags);
907 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
909 if (atomic_dec_and_test(&io->io_count)) {
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200910 if (io->status == BLK_STS_DM_REQUEUE) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800911 /*
912 * Target requested pushing back the I/O.
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800913 */
Mikulas Patocka022c2612009-04-02 19:55:39 +0100914 spin_lock_irqsave(&md->deferred_lock, flags);
Tejun Heo6a8736d2010-09-08 18:07:00 +0200915 if (__noflush_suspending(md))
Mike Snitzer745dc572017-12-11 20:51:50 -0500916 /* NOTE early return due to BLK_STS_DM_REQUEUE below */
917 bio_list_add_head(&md->deferred, io->orig_bio);
Tejun Heo6a8736d2010-09-08 18:07:00 +0200918 else
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800919 /* noflush suspend was interrupted. */
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200920 io->status = BLK_STS_IOERR;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100921 spin_unlock_irqrestore(&md->deferred_lock, flags);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800922 }
923
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200924 io_error = io->status;
Mike Snitzer745dc572017-12-11 20:51:50 -0500925 bio = io->orig_bio;
Tejun Heo6a8736d2010-09-08 18:07:00 +0200926 end_io_acct(io);
927 free_io(md, io);
Jens Axboe2056a782006-03-23 20:00:26 +0100928
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200929 if (io_error == BLK_STS_DM_REQUEUE)
Tejun Heo6a8736d2010-09-08 18:07:00 +0200930 return;
931
Jens Axboe1eff9d32016-08-05 15:35:16 -0600932 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100933 /*
Tejun Heo6a8736d2010-09-08 18:07:00 +0200934 * Preflush done for flush with data, reissue
Mike Christie28a8f0d2016-06-05 14:32:25 -0500935 * without REQ_PREFLUSH.
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100936 */
Jens Axboe1eff9d32016-08-05 15:35:16 -0600937 bio->bi_opf &= ~REQ_PREFLUSH;
Tejun Heo6a8736d2010-09-08 18:07:00 +0200938 queue_io(md, bio);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100939 } else {
Mike Snitzerb372d362010-09-08 18:07:01 +0200940 /* done with normal IO or empty flush */
NeilBrown8dd601f2018-02-15 20:00:15 +1100941 if (io_error)
942 bio->bi_status = io_error;
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200943 bio_endio(bio);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800944 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945 }
946}
947
Mike Snitzerbcb44432019-04-03 12:23:11 -0400948void disable_discard(struct mapped_device *md)
949{
950 struct queue_limits *limits = dm_get_queue_limits(md);
951
952 /* device doesn't really support DISCARD, disable it */
953 limits->max_discard_sectors = 0;
954 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
955}
956
Mike Snitzer4cc96132016-05-12 16:28:10 -0400957void disable_write_same(struct mapped_device *md)
Mike Snitzer7eee4ae2014-06-02 15:50:06 -0400958{
959 struct queue_limits *limits = dm_get_queue_limits(md);
960
961 /* device doesn't really support WRITE SAME, disable it */
962 limits->max_write_same_sectors = 0;
963}
964
Christoph Hellwigac62d622017-04-05 19:21:05 +0200965void disable_write_zeroes(struct mapped_device *md)
966{
967 struct queue_limits *limits = dm_get_queue_limits(md);
968
969 /* device doesn't really support WRITE ZEROES, disable it */
970 limits->max_write_zeroes_sectors = 0;
971}
972
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200973static void clone_endio(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974{
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200975 blk_status_t error = bio->bi_status;
Mikulas Patockabfc6d412014-03-04 18:24:49 -0500976 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000977 struct dm_io *io = tio->io;
Stefan Bader9faf4002006-10-03 01:15:41 -0700978 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 dm_endio_fn endio = tio->ti->type->end_io;
980
Mike Snitzer978e51b2017-12-09 15:16:42 -0500981 if (unlikely(error == BLK_STS_TARGET) && md->type != DM_TYPE_NVME_BIO_BASED) {
Mike Snitzerbcb44432019-04-03 12:23:11 -0400982 if (bio_op(bio) == REQ_OP_DISCARD &&
983 !bio->bi_disk->queue->limits.max_discard_sectors)
984 disable_discard(md);
985 else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
986 !bio->bi_disk->queue->limits.max_write_same_sectors)
Christoph Hellwigac62d622017-04-05 19:21:05 +0200987 disable_write_same(md);
Mike Snitzerbcb44432019-04-03 12:23:11 -0400988 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
989 !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
Christoph Hellwigac62d622017-04-05 19:21:05 +0200990 disable_write_zeroes(md);
991 }
Mike Snitzer7eee4ae2014-06-02 15:50:06 -0400992
Christoph Hellwig1be56902017-06-03 09:38:03 +0200993 if (endio) {
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200994 int r = endio(tio->ti, bio, &error);
Christoph Hellwig1be56902017-06-03 09:38:03 +0200995 switch (r) {
996 case DM_ENDIO_REQUEUE:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200997 error = BLK_STS_DM_REQUEUE;
Christoph Hellwig1be56902017-06-03 09:38:03 +0200998 /*FALLTHRU*/
999 case DM_ENDIO_DONE:
1000 break;
1001 case DM_ENDIO_INCOMPLETE:
1002 /* The target will handle the io */
1003 return;
1004 default:
1005 DMWARN("unimplemented target endio return value: %d", r);
1006 BUG();
1007 }
1008 }
1009
Mike Snitzercfae7522016-04-11 12:05:38 -04001010 free_tio(tio);
Milan Brozb35f8ca2009-03-16 17:44:36 +00001011 dec_pending(io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012}
1013
Mike Snitzer78d8e582015-06-26 10:01:13 -04001014/*
Mike Snitzer56a67df2010-08-12 04:14:10 +01001015 * Return maximum size of I/O possible at the supplied sector up to the current
1016 * target boundary.
1017 */
1018static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019{
Mike Snitzer56a67df2010-08-12 04:14:10 +01001020 sector_t target_offset = dm_target_offset(ti, sector);
1021
1022 return ti->len - target_offset;
1023}
1024
1025static sector_t max_io_len(sector_t sector, struct dm_target *ti)
1026{
1027 sector_t len = max_io_len_target_boundary(sector, ti);
Mike Snitzer542f9032012-07-27 15:08:00 +01001028 sector_t offset, max_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029
1030 /*
Mike Snitzer542f9032012-07-27 15:08:00 +01001031 * Does the target need to split even further?
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 */
Mike Snitzer542f9032012-07-27 15:08:00 +01001033 if (ti->max_io_len) {
1034 offset = dm_target_offset(ti, sector);
1035 if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
1036 max_len = sector_div(offset, ti->max_io_len);
1037 else
1038 max_len = offset & (ti->max_io_len - 1);
1039 max_len = ti->max_io_len - max_len;
1040
1041 if (len > max_len)
1042 len = max_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 }
1044
1045 return len;
1046}
1047
Mike Snitzer542f9032012-07-27 15:08:00 +01001048int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1049{
1050 if (len > UINT_MAX) {
1051 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1052 (unsigned long long)len, UINT_MAX);
1053 ti->error = "Maximum size of target IO is too large";
1054 return -EINVAL;
1055 }
1056
Mikulas Patocka75ae1932019-03-21 16:46:12 -04001057 ti->max_io_len = (uint32_t) len;
Mike Snitzer542f9032012-07-27 15:08:00 +01001058
1059 return 0;
1060}
1061EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1062
Dan Williamsf26c5712017-04-12 12:35:44 -07001063static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
Mike Snitzer3d97c822018-04-30 16:06:28 -04001064 sector_t sector, int *srcu_idx)
1065 __acquires(md->io_barrier)
Toshi Kani545ed202016-06-22 17:54:53 -06001066{
Toshi Kani545ed202016-06-22 17:54:53 -06001067 struct dm_table *map;
1068 struct dm_target *ti;
Toshi Kani545ed202016-06-22 17:54:53 -06001069
Dan Williamsf26c5712017-04-12 12:35:44 -07001070 map = dm_get_live_table(md, srcu_idx);
Toshi Kani545ed202016-06-22 17:54:53 -06001071 if (!map)
Dan Williamsf26c5712017-04-12 12:35:44 -07001072 return NULL;
Toshi Kani545ed202016-06-22 17:54:53 -06001073
1074 ti = dm_table_find_target(map, sector);
Mikulas Patocka123d87d2019-08-23 09:55:26 -04001075 if (!ti)
Dan Williamsf26c5712017-04-12 12:35:44 -07001076 return NULL;
1077
1078 return ti;
1079}
1080
1081static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
Mike Snitzer3d97c822018-04-30 16:06:28 -04001082 long nr_pages, void **kaddr, pfn_t *pfn)
Dan Williamsf26c5712017-04-12 12:35:44 -07001083{
1084 struct mapped_device *md = dax_get_private(dax_dev);
1085 sector_t sector = pgoff * PAGE_SECTORS;
1086 struct dm_target *ti;
1087 long len, ret = -EIO;
1088 int srcu_idx;
1089
1090 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1091
1092 if (!ti)
Toshi Kani545ed202016-06-22 17:54:53 -06001093 goto out;
Dan Williamsf26c5712017-04-12 12:35:44 -07001094 if (!ti->type->direct_access)
1095 goto out;
1096 len = max_io_len(sector, ti) / PAGE_SECTORS;
1097 if (len < 1)
1098 goto out;
1099 nr_pages = min(len, nr_pages);
Ross Zwislerdbc62652018-06-26 16:30:41 -06001100 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
Dan Williams817bf402017-04-12 13:37:44 -07001101
Dan Williamsf26c5712017-04-12 12:35:44 -07001102 out:
Toshi Kani545ed202016-06-22 17:54:53 -06001103 dm_put_live_table(md, srcu_idx);
Dan Williamsf26c5712017-04-12 12:35:44 -07001104
1105 return ret;
Toshi Kani545ed202016-06-22 17:54:53 -06001106}
1107
Dan Williams7bf7eac2019-05-16 13:26:29 -07001108static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bdev,
1109 int blocksize, sector_t start, sector_t len)
1110{
1111 struct mapped_device *md = dax_get_private(dax_dev);
1112 struct dm_table *map;
1113 int srcu_idx;
1114 bool ret;
1115
1116 map = dm_get_live_table(md, &srcu_idx);
1117 if (!map)
1118 return false;
1119
Pankaj Gupta2e9ee092019-07-05 19:33:25 +05301120 ret = dm_table_supports_dax(map, device_supports_dax, &blocksize);
Dan Williams7bf7eac2019-05-16 13:26:29 -07001121
1122 dm_put_live_table(md, srcu_idx);
1123
1124 return ret;
1125}
1126
Dan Williams7e026c82017-05-29 12:57:56 -07001127static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
Mike Snitzer3d97c822018-04-30 16:06:28 -04001128 void *addr, size_t bytes, struct iov_iter *i)
Dan Williams7e026c82017-05-29 12:57:56 -07001129{
1130 struct mapped_device *md = dax_get_private(dax_dev);
1131 sector_t sector = pgoff * PAGE_SECTORS;
1132 struct dm_target *ti;
1133 long ret = 0;
1134 int srcu_idx;
1135
1136 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1137
1138 if (!ti)
1139 goto out;
1140 if (!ti->type->dax_copy_from_iter) {
1141 ret = copy_from_iter(addr, bytes, i);
1142 goto out;
1143 }
1144 ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
1145 out:
1146 dm_put_live_table(md, srcu_idx);
1147
1148 return ret;
1149}
1150
Dan Williamsb3a9a0c2018-05-02 06:46:33 -07001151static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1152 void *addr, size_t bytes, struct iov_iter *i)
1153{
1154 struct mapped_device *md = dax_get_private(dax_dev);
1155 sector_t sector = pgoff * PAGE_SECTORS;
1156 struct dm_target *ti;
1157 long ret = 0;
1158 int srcu_idx;
1159
1160 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1161
1162 if (!ti)
1163 goto out;
1164 if (!ti->type->dax_copy_to_iter) {
1165 ret = copy_to_iter(addr, bytes, i);
1166 goto out;
1167 }
1168 ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
1169 out:
1170 dm_put_live_table(md, srcu_idx);
1171
1172 return ret;
1173}
1174
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001175/*
1176 * A target may call dm_accept_partial_bio only from the map routine. It is
Ajay Joshi2e2d6f72019-10-27 23:05:48 +09001177 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_RESET,
1178 * REQ_OP_ZONE_OPEN, REQ_OP_ZONE_CLOSE and REQ_OP_ZONE_FINISH.
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001179 *
1180 * dm_accept_partial_bio informs the dm that the target only wants to process
1181 * additional n_sectors sectors of the bio and the rest of the data should be
1182 * sent in a next bio.
1183 *
1184 * A diagram that explains the arithmetics:
1185 * +--------------------+---------------+-------+
1186 * | 1 | 2 | 3 |
1187 * +--------------------+---------------+-------+
1188 *
1189 * <-------------- *tio->len_ptr --------------->
1190 * <------- bi_size ------->
1191 * <-- n_sectors -->
1192 *
1193 * Region 1 was already iterated over with bio_advance or similar function.
1194 * (it may be empty if the target doesn't use bio_advance)
1195 * Region 2 is the remaining bio size that the target wants to process.
1196 * (it may be empty if region 1 is non-empty, although there is no reason
1197 * to make it empty)
1198 * The target requires that region 3 is to be sent in the next bio.
1199 *
1200 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1201 * the partially processed part (the sum of regions 1+2) must be the same for all
1202 * copies of the bio.
1203 */
1204void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1205{
1206 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1207 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
Jens Axboe1eff9d32016-08-05 15:35:16 -06001208 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001209 BUG_ON(bi_size > *tio->len_ptr);
1210 BUG_ON(n_sectors > bi_size);
1211 *tio->len_ptr -= bi_size - n_sectors;
1212 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1213}
1214EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1215
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001216/*
Christoph Hellwige76239a2018-10-12 19:08:49 +09001217 * The zone descriptors obtained with a zone report indicate
1218 * zone positions within the underlying device of the target. The zone
1219 * descriptors must be remapped to match their position within the dm device.
1220 * The caller target should obtain the zones information using
1221 * blkdev_report_zones() to ensure that remapping for partition offset is
1222 * already handled.
Damien Le Moal10999302017-05-08 16:40:48 -07001223 */
Christoph Hellwige76239a2018-10-12 19:08:49 +09001224void dm_remap_zone_report(struct dm_target *ti, sector_t start,
1225 struct blk_zone *zones, unsigned int *nr_zones)
Damien Le Moal10999302017-05-08 16:40:48 -07001226{
1227#ifdef CONFIG_BLK_DEV_ZONED
Damien Le Moal10999302017-05-08 16:40:48 -07001228 struct blk_zone *zone;
Christoph Hellwige76239a2018-10-12 19:08:49 +09001229 unsigned int nrz = *nr_zones;
1230 int i;
Damien Le Moal10999302017-05-08 16:40:48 -07001231
1232 /*
Christoph Hellwige76239a2018-10-12 19:08:49 +09001233 * Remap the start sector and write pointer position of the zones in
1234 * the array. Since we may have obtained from the target underlying
1235 * device more zones that the target size, also adjust the number
1236 * of zones.
Damien Le Moal9864cd52018-10-09 14:24:31 +09001237 */
Christoph Hellwige76239a2018-10-12 19:08:49 +09001238 for (i = 0; i < nrz; i++) {
1239 zone = zones + i;
1240 if (zone->start >= start + ti->len) {
1241 memset(zone, 0, sizeof(struct blk_zone) * (nrz - i));
1242 break;
Damien Le Moal10999302017-05-08 16:40:48 -07001243 }
1244
Christoph Hellwige76239a2018-10-12 19:08:49 +09001245 zone->start = zone->start + ti->begin - start;
1246 if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
1247 continue;
Damien Le Moal10999302017-05-08 16:40:48 -07001248
Christoph Hellwige76239a2018-10-12 19:08:49 +09001249 if (zone->cond == BLK_ZONE_COND_FULL)
1250 zone->wp = zone->start + zone->len;
1251 else if (zone->cond == BLK_ZONE_COND_EMPTY)
1252 zone->wp = zone->start;
1253 else
1254 zone->wp = zone->wp + ti->begin - start;
Damien Le Moal10999302017-05-08 16:40:48 -07001255 }
1256
Christoph Hellwige76239a2018-10-12 19:08:49 +09001257 *nr_zones = i;
Damien Le Moal10999302017-05-08 16:40:48 -07001258#else /* !CONFIG_BLK_DEV_ZONED */
Christoph Hellwige76239a2018-10-12 19:08:49 +09001259 *nr_zones = 0;
Damien Le Moal10999302017-05-08 16:40:48 -07001260#endif
1261}
1262EXPORT_SYMBOL_GPL(dm_remap_zone_report);
1263
Mike Snitzer978e51b2017-12-09 15:16:42 -05001264static blk_qc_t __map_bio(struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265{
1266 int r;
Jens Axboe2056a782006-03-23 20:00:26 +01001267 sector_t sector;
Mikulas Patockadba14162012-10-12 21:02:15 +01001268 struct bio *clone = &tio->clone;
Mike Snitzer64f52b02017-12-11 23:17:47 -05001269 struct dm_io *io = tio->io;
Mike Snitzer978e51b2017-12-09 15:16:42 -05001270 struct mapped_device *md = io->md;
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001271 struct dm_target *ti = tio->ti;
Mike Snitzer978e51b2017-12-09 15:16:42 -05001272 blk_qc_t ret = BLK_QC_T_NONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274 clone->bi_end_io = clone_endio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275
1276 /*
1277 * Map the clone. If r == 0 we don't need to do
1278 * anything, the target has assumed ownership of
1279 * this io.
1280 */
Mike Snitzer64f52b02017-12-11 23:17:47 -05001281 atomic_inc(&io->io_count);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001282 sector = clone->bi_iter.bi_sector;
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001283
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00001284 r = ti->type->map(ti, clone);
Christoph Hellwig846785e2017-06-03 09:38:02 +02001285 switch (r) {
1286 case DM_MAPIO_SUBMITTED:
1287 break;
1288 case DM_MAPIO_REMAPPED:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289 /* the bio has been remapped so dispatch it */
Christoph Hellwig74d46992017-08-23 19:10:32 +02001290 trace_block_bio_remap(clone->bi_disk->queue, clone,
Mike Snitzer64f52b02017-12-11 23:17:47 -05001291 bio_dev(io->orig_bio), sector);
Mike Snitzer978e51b2017-12-09 15:16:42 -05001292 if (md->type == DM_TYPE_NVME_BIO_BASED)
1293 ret = direct_make_request(clone);
1294 else
1295 ret = generic_make_request(clone);
Christoph Hellwig846785e2017-06-03 09:38:02 +02001296 break;
1297 case DM_MAPIO_KILL:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02001298 free_tio(tio);
Mike Snitzer64f52b02017-12-11 23:17:47 -05001299 dec_pending(io, BLK_STS_IOERR);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02001300 break;
Christoph Hellwig846785e2017-06-03 09:38:02 +02001301 case DM_MAPIO_REQUEUE:
Mike Snitzercfae7522016-04-11 12:05:38 -04001302 free_tio(tio);
Mike Snitzer64f52b02017-12-11 23:17:47 -05001303 dec_pending(io, BLK_STS_DM_REQUEUE);
Christoph Hellwig846785e2017-06-03 09:38:02 +02001304 break;
1305 default:
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -08001306 DMWARN("unimplemented target map return value: %d", r);
1307 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001308 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309
Mike Snitzer978e51b2017-12-09 15:16:42 -05001310 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Mikulas Patockae0d66092014-03-14 18:40:39 -04001313static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001314{
Kent Overstreet4f024f32013-10-11 15:44:27 -07001315 bio->bi_iter.bi_sector = sector;
1316 bio->bi_iter.bi_size = to_bytes(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317}
1318
1319/*
1320 * Creates a bio that consists of range of complete bvecs.
1321 */
Mike Snitzerc80914e2016-03-02 12:33:03 -05001322static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1323 sector_t sector, unsigned len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324{
Mikulas Patockadba14162012-10-12 21:02:15 +01001325 struct bio *clone = &tio->clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001327 __bio_clone_fast(clone, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328
Mike Snitzer57c36512019-01-16 18:53:26 -05001329 if (bio_integrity(bio)) {
Mikulas Patockae2460f22017-04-18 16:51:48 -04001330 int r;
1331
1332 if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
1333 !dm_target_passes_integrity(tio->ti->type))) {
1334 DMWARN("%s: the target %s doesn't support integrity data.",
1335 dm_device_name(tio->io->md),
1336 tio->ti->type->name);
1337 return -EIO;
1338 }
1339
1340 r = bio_integrity_clone(clone, bio, GFP_NOIO);
Mike Snitzerc80914e2016-03-02 12:33:03 -05001341 if (r < 0)
1342 return r;
1343 }
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001344
Mike Snitzerfa8db492019-02-05 17:07:58 -05001345 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1346 clone->bi_iter.bi_size = to_bytes(len);
1347
1348 if (bio_integrity(bio))
1349 bio_integrity_trim(clone);
Mike Snitzerc80914e2016-03-02 12:33:03 -05001350
1351 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352}
1353
Mike Snitzer318716d2017-11-22 14:56:12 -05001354static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1355 struct dm_target *ti, unsigned num_bios)
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001356{
Mikulas Patockadba14162012-10-12 21:02:15 +01001357 struct dm_target_io *tio;
Mike Snitzer318716d2017-11-22 14:56:12 -05001358 int try;
Mikulas Patockadba14162012-10-12 21:02:15 +01001359
Mike Snitzer318716d2017-11-22 14:56:12 -05001360 if (!num_bios)
1361 return;
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001362
Mike Snitzer318716d2017-11-22 14:56:12 -05001363 if (num_bios == 1) {
1364 tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1365 bio_list_add(blist, &tio->clone);
1366 return;
1367 }
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001368
Mike Snitzer318716d2017-11-22 14:56:12 -05001369 for (try = 0; try < 2; try++) {
1370 int bio_nr;
1371 struct bio *bio;
1372
1373 if (try)
Mike Snitzerbc02cdb2017-12-14 16:30:42 -05001374 mutex_lock(&ci->io->md->table_devices_lock);
Mike Snitzer318716d2017-11-22 14:56:12 -05001375 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1376 tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
1377 if (!tio)
1378 break;
1379
1380 bio_list_add(blist, &tio->clone);
1381 }
1382 if (try)
Mike Snitzerbc02cdb2017-12-14 16:30:42 -05001383 mutex_unlock(&ci->io->md->table_devices_lock);
Mike Snitzer318716d2017-11-22 14:56:12 -05001384 if (bio_nr == num_bios)
1385 return;
1386
1387 while ((bio = bio_list_pop(blist))) {
1388 tio = container_of(bio, struct dm_target_io, clone);
1389 free_tio(tio);
1390 }
1391 }
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001392}
1393
Mike Snitzer978e51b2017-12-09 15:16:42 -05001394static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
1395 struct dm_target_io *tio, unsigned *len)
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001396{
Mikulas Patockadba14162012-10-12 21:02:15 +01001397 struct bio *clone = &tio->clone;
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001398
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001399 tio->len_ptr = len;
1400
Junichi Nomura99778272014-10-03 11:55:16 +00001401 __bio_clone_fast(clone, ci->bio);
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001402 if (len)
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001403 bio_setup_sector(clone, ci->sector, *len);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001404
Mike Snitzer978e51b2017-12-09 15:16:42 -05001405 return __map_bio(tio);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001406}
1407
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001408static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001409 unsigned num_bios, unsigned *len)
Mike Snitzer06a426c2010-08-12 04:14:09 +01001410{
Mike Snitzer318716d2017-11-22 14:56:12 -05001411 struct bio_list blist = BIO_EMPTY_LIST;
1412 struct bio *bio;
1413 struct dm_target_io *tio;
Mike Snitzer06a426c2010-08-12 04:14:09 +01001414
Mike Snitzer318716d2017-11-22 14:56:12 -05001415 alloc_multiple_bios(&blist, ci, ti, num_bios);
1416
1417 while ((bio = bio_list_pop(&blist))) {
1418 tio = container_of(bio, struct dm_target_io, clone);
Mike Snitzer978e51b2017-12-09 15:16:42 -05001419 (void) __clone_and_map_simple_bio(ci, tio, len);
Mike Snitzer318716d2017-11-22 14:56:12 -05001420 }
Mike Snitzer06a426c2010-08-12 04:14:09 +01001421}
1422
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001423static int __send_empty_flush(struct clone_info *ci)
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001424{
Mike Snitzer06a426c2010-08-12 04:14:09 +01001425 unsigned target_nr = 0;
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001426 struct dm_target *ti;
1427
Dennis Zhou892ad712018-12-05 12:10:30 -05001428 /*
Jens Axboedbe3ece2018-12-19 09:13:34 -07001429 * Empty flush uses a statically initialized bio, as the base for
1430 * cloning. However, blkg association requires that a bdev is
1431 * associated with a gendisk, which doesn't happen until the bdev is
1432 * opened. So, blkg association is done at issue time of the flush
1433 * rather than when the device is created in alloc_dev().
Dennis Zhou892ad712018-12-05 12:10:30 -05001434 */
1435 bio_set_dev(ci->bio, ci->io->md->bdev);
1436
Mike Snitzerb372d362010-09-08 18:07:01 +02001437 BUG_ON(bio_has_data(ci->bio));
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001438 while ((ti = dm_table_get_target(ci->map, target_nr++)))
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001439 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001440
Dennis Zhou892ad712018-12-05 12:10:30 -05001441 bio_disassociate_blkg(ci->bio);
1442
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001443 return 0;
1444}
1445
Mike Snitzerc80914e2016-03-02 12:33:03 -05001446static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
NeilBrownf31c21e2017-11-22 14:25:18 +11001447 sector_t sector, unsigned *len)
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001448{
Mikulas Patockadba14162012-10-12 21:02:15 +01001449 struct bio *bio = ci->bio;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001450 struct dm_target_io *tio;
NeilBrownf31c21e2017-11-22 14:25:18 +11001451 int r;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001452
Mike Snitzer318716d2017-11-22 14:56:12 -05001453 tio = alloc_tio(ci, ti, 0, GFP_NOIO);
NeilBrownf31c21e2017-11-22 14:25:18 +11001454 tio->len_ptr = len;
1455 r = clone_bio(tio, bio, sector, *len);
1456 if (r < 0) {
1457 free_tio(tio);
1458 return r;
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001459 }
Mike Snitzer978e51b2017-12-09 15:16:42 -05001460 (void) __map_bio(tio);
Mike Snitzerc80914e2016-03-02 12:33:03 -05001461
NeilBrownf31c21e2017-11-22 14:25:18 +11001462 return 0;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001463}
1464
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001465typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
Mike Snitzer23508a92012-12-21 20:23:37 +00001466
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001467static unsigned get_num_discard_bios(struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001468{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001469 return ti->num_discard_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001470}
1471
Denis Semakin00716542018-03-13 13:23:45 +04001472static unsigned get_num_secure_erase_bios(struct dm_target *ti)
1473{
1474 return ti->num_secure_erase_bios;
1475}
1476
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001477static unsigned get_num_write_same_bios(struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001478{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001479 return ti->num_write_same_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001480}
1481
Christoph Hellwigac62d622017-04-05 19:21:05 +02001482static unsigned get_num_write_zeroes_bios(struct dm_target *ti)
1483{
1484 return ti->num_write_zeroes_bios;
1485}
1486
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001487static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
Mike Snitzer61697a62019-01-18 14:19:26 -05001488 unsigned num_bios)
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001489{
Michael Lass51b86f92019-05-21 21:58:07 +02001490 unsigned len;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001491
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001492 /*
1493 * Even though the device advertised support for this type of
1494 * request, that does not mean every target supports it, and
1495 * reconfiguration might also have changed that since the
1496 * check was performed.
1497 */
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001498 if (!num_bios)
1499 return -EOPNOTSUPP;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001500
Michael Lass51b86f92019-05-21 21:58:07 +02001501 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1502
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001503 __send_duplicate_bios(ci, ti, num_bios, &len);
Mike Snitzer06a426c2010-08-12 04:14:09 +01001504
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001505 ci->sector += len;
1506 ci->sector_count -= len;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001507
1508 return 0;
1509}
1510
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001511static int __send_discard(struct clone_info *ci, struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001512{
Mike Snitzer61697a62019-01-18 14:19:26 -05001513 return __send_changing_extent_only(ci, ti, get_num_discard_bios(ti));
Mike Snitzer23508a92012-12-21 20:23:37 +00001514}
1515
Denis Semakin00716542018-03-13 13:23:45 +04001516static int __send_secure_erase(struct clone_info *ci, struct dm_target *ti)
1517{
Mike Snitzer61697a62019-01-18 14:19:26 -05001518 return __send_changing_extent_only(ci, ti, get_num_secure_erase_bios(ti));
Denis Semakin00716542018-03-13 13:23:45 +04001519}
1520
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001521static int __send_write_same(struct clone_info *ci, struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001522{
Mike Snitzer61697a62019-01-18 14:19:26 -05001523 return __send_changing_extent_only(ci, ti, get_num_write_same_bios(ti));
Mike Snitzer23508a92012-12-21 20:23:37 +00001524}
1525
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001526static int __send_write_zeroes(struct clone_info *ci, struct dm_target *ti)
Christoph Hellwigac62d622017-04-05 19:21:05 +02001527{
Mike Snitzer61697a62019-01-18 14:19:26 -05001528 return __send_changing_extent_only(ci, ti, get_num_write_zeroes_bios(ti));
Christoph Hellwigac62d622017-04-05 19:21:05 +02001529}
1530
Mike Snitzer568c73a2019-01-18 14:10:37 -05001531static bool is_abnormal_io(struct bio *bio)
1532{
1533 bool r = false;
1534
1535 switch (bio_op(bio)) {
1536 case REQ_OP_DISCARD:
1537 case REQ_OP_SECURE_ERASE:
1538 case REQ_OP_WRITE_SAME:
1539 case REQ_OP_WRITE_ZEROES:
1540 r = true;
1541 break;
1542 }
1543
1544 return r;
1545}
1546
Mike Snitzer0519c712018-03-26 11:49:16 -04001547static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1548 int *result)
1549{
1550 struct bio *bio = ci->bio;
1551
1552 if (bio_op(bio) == REQ_OP_DISCARD)
1553 *result = __send_discard(ci, ti);
Denis Semakin00716542018-03-13 13:23:45 +04001554 else if (bio_op(bio) == REQ_OP_SECURE_ERASE)
1555 *result = __send_secure_erase(ci, ti);
Mike Snitzer0519c712018-03-26 11:49:16 -04001556 else if (bio_op(bio) == REQ_OP_WRITE_SAME)
1557 *result = __send_write_same(ci, ti);
1558 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
1559 *result = __send_write_zeroes(ci, ti);
1560 else
1561 return false;
1562
1563 return true;
1564}
1565
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001566/*
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001567 * Select the correct strategy for processing a non-flush bio.
1568 */
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001569static int __split_and_process_non_flush(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570{
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001571 struct dm_target *ti;
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001572 unsigned len;
Mike Snitzerc80914e2016-03-02 12:33:03 -05001573 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001575 ti = dm_table_find_target(ci->map, ci->sector);
Mikulas Patocka123d87d2019-08-23 09:55:26 -04001576 if (!ti)
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001577 return -EIO;
1578
Mike Snitzer568c73a2019-01-18 14:10:37 -05001579 if (__process_abnormal_io(ci, ti, &r))
Mike Snitzer0519c712018-03-26 11:49:16 -04001580 return r;
Mike Snitzer3d7f4562017-12-08 15:02:11 -05001581
Christoph Hellwige76239a2018-10-12 19:08:49 +09001582 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001583
Mike Snitzerc80914e2016-03-02 12:33:03 -05001584 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1585 if (r < 0)
1586 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001588 ci->sector += len;
1589 ci->sector_count -= len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001591 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592}
1593
Mike Snitzer978e51b2017-12-09 15:16:42 -05001594static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1595 struct dm_table *map, struct bio *bio)
1596{
1597 ci->map = map;
1598 ci->io = alloc_io(md, bio);
1599 ci->sector = bio->bi_iter.bi_sector;
1600}
1601
Mike Snitzera1e1cb72019-01-17 10:48:01 -05001602#define __dm_part_stat_sub(part, field, subnd) \
1603 (part_stat_get(part, field) -= (subnd))
1604
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605/*
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001606 * Entry point to split a bio into clones and submit them to the targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607 */
Mike Snitzer978e51b2017-12-09 15:16:42 -05001608static blk_qc_t __split_and_process_bio(struct mapped_device *md,
1609 struct dm_table *map, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610{
1611 struct clone_info ci;
Mike Snitzer978e51b2017-12-09 15:16:42 -05001612 blk_qc_t ret = BLK_QC_T_NONE;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001613 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614
Mike Snitzer978e51b2017-12-09 15:16:42 -05001615 init_clone_info(&ci, md, map, bio);
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001616
Jens Axboe1eff9d32016-08-05 15:35:16 -06001617 if (bio->bi_opf & REQ_PREFLUSH) {
Jens Axboedbe3ece2018-12-19 09:13:34 -07001618 struct bio flush_bio;
1619
1620 /*
1621 * Use an on-stack bio for this, it's safe since we don't
1622 * need to reference it after submit. It's just used as
1623 * the basis for the clone(s).
1624 */
1625 bio_init(&flush_bio, NULL, 0);
1626 flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1627 ci.bio = &flush_bio;
Mike Snitzerb372d362010-09-08 18:07:01 +02001628 ci.sector_count = 0;
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001629 error = __send_empty_flush(&ci);
Mike Snitzerb372d362010-09-08 18:07:01 +02001630 /* dec_pending submits any data associated with flush */
Ajay Joshi2e2d6f72019-10-27 23:05:48 +09001631 } else if (op_is_zone_mgmt(bio_op(bio))) {
Damien Le Moala4aa5e52017-05-08 16:40:46 -07001632 ci.bio = bio;
1633 ci.sector_count = 0;
1634 error = __split_and_process_non_flush(&ci);
Mike Snitzerb372d362010-09-08 18:07:01 +02001635 } else {
Tejun Heo6a8736d2010-09-08 18:07:00 +02001636 ci.bio = bio;
Tejun Heod87f4c12010-09-03 11:56:19 +02001637 ci.sector_count = bio_sectors(bio);
NeilBrown18a25da2017-09-06 09:43:28 +10001638 while (ci.sector_count && !error) {
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001639 error = __split_and_process_non_flush(&ci);
NeilBrown18a25da2017-09-06 09:43:28 +10001640 if (current->bio_list && ci.sector_count && !error) {
1641 /*
1642 * Remainder must be passed to generic_make_request()
1643 * so that it gets handled *after* bios already submitted
1644 * have been completely processed.
1645 * We take a clone of the original to store in
Mike Snitzer745dc572017-12-11 20:51:50 -05001646 * ci.io->orig_bio to be used by end_io_acct() and
NeilBrown18a25da2017-09-06 09:43:28 +10001647 * for dec_pending to use for completion handling.
NeilBrown18a25da2017-09-06 09:43:28 +10001648 */
Mike Snitzerf21c6012018-06-15 09:35:33 -04001649 struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1650 GFP_NOIO, &md->queue->bio_split);
Mike Snitzer745dc572017-12-11 20:51:50 -05001651 ci.io->orig_bio = b;
Mike Snitzera1e1cb72019-01-17 10:48:01 -05001652
1653 /*
1654 * Adjust IO stats for each split, otherwise upon queue
1655 * reentry there will be redundant IO accounting.
1656 * NOTE: this is a stop-gap fix, a proper fix involves
1657 * significant refactoring of DM core's bio splitting
1658 * (by eliminating DM's splitting and just using bio_split)
1659 */
1660 part_stat_lock();
1661 __dm_part_stat_sub(&dm_disk(md)->part0,
1662 sectors[op_stat_group(bio_op(bio))], ci.sector_count);
1663 part_stat_unlock();
1664
NeilBrown18a25da2017-09-06 09:43:28 +10001665 bio_chain(b, bio);
Mike Snitzer075c18c32019-01-18 01:21:11 -05001666 trace_block_split(md->queue, b, bio->bi_iter.bi_sector);
Mike Snitzer978e51b2017-12-09 15:16:42 -05001667 ret = generic_make_request(bio);
NeilBrown18a25da2017-09-06 09:43:28 +10001668 break;
1669 }
1670 }
Tejun Heod87f4c12010-09-03 11:56:19 +02001671 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672
1673 /* drop the extra reference count */
Bart Van Assche54385bf2017-08-09 11:32:10 -07001674 dec_pending(ci.io, errno_to_blk_status(error));
Mike Snitzer978e51b2017-12-09 15:16:42 -05001675 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677
1678/*
Mike Snitzer978e51b2017-12-09 15:16:42 -05001679 * Optimized variant of __split_and_process_bio that leverages the
1680 * fact that targets that use it do _not_ have a need to split bios.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681 */
Mike Snitzer568c73a2019-01-18 14:10:37 -05001682static blk_qc_t __process_bio(struct mapped_device *md, struct dm_table *map,
1683 struct bio *bio, struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684{
Mike Snitzer978e51b2017-12-09 15:16:42 -05001685 struct clone_info ci;
1686 blk_qc_t ret = BLK_QC_T_NONE;
1687 int error = 0;
1688
Mike Snitzer978e51b2017-12-09 15:16:42 -05001689 init_clone_info(&ci, md, map, bio);
1690
1691 if (bio->bi_opf & REQ_PREFLUSH) {
Jens Axboedbe3ece2018-12-19 09:13:34 -07001692 struct bio flush_bio;
1693
1694 /*
1695 * Use an on-stack bio for this, it's safe since we don't
1696 * need to reference it after submit. It's just used as
1697 * the basis for the clone(s).
1698 */
1699 bio_init(&flush_bio, NULL, 0);
1700 flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1701 ci.bio = &flush_bio;
Mike Snitzer978e51b2017-12-09 15:16:42 -05001702 ci.sector_count = 0;
1703 error = __send_empty_flush(&ci);
1704 /* dec_pending submits any data associated with flush */
1705 } else {
Mike Snitzer978e51b2017-12-09 15:16:42 -05001706 struct dm_target_io *tio;
1707
Mike Snitzer978e51b2017-12-09 15:16:42 -05001708 ci.bio = bio;
1709 ci.sector_count = bio_sectors(bio);
Mike Snitzer568c73a2019-01-18 14:10:37 -05001710 if (__process_abnormal_io(&ci, ti, &error))
Mike Snitzer0519c712018-03-26 11:49:16 -04001711 goto out;
1712
1713 tio = alloc_tio(&ci, ti, 0, GFP_NOIO);
Mike Snitzer978e51b2017-12-09 15:16:42 -05001714 ret = __clone_and_map_simple_bio(&ci, tio, NULL);
1715 }
1716out:
1717 /* drop the extra reference count */
1718 dec_pending(ci.io, errno_to_blk_status(error));
1719 return ret;
1720}
1721
Mike Snitzer568c73a2019-01-18 14:10:37 -05001722static void dm_queue_split(struct mapped_device *md, struct dm_target *ti, struct bio **bio)
1723{
1724 unsigned len, sector_count;
1725
1726 sector_count = bio_sectors(*bio);
1727 len = min_t(sector_t, max_io_len((*bio)->bi_iter.bi_sector, ti), sector_count);
1728
1729 if (sector_count > len) {
1730 struct bio *split = bio_split(*bio, len, GFP_NOIO, &md->queue->bio_split);
1731
1732 bio_chain(split, *bio);
1733 trace_block_split(md->queue, split, (*bio)->bi_iter.bi_sector);
1734 generic_make_request(*bio);
1735 *bio = split;
1736 }
1737}
1738
Mike Snitzer6548c7c2019-01-17 14:33:01 -05001739static blk_qc_t dm_process_bio(struct mapped_device *md,
1740 struct dm_table *map, struct bio *bio)
1741{
Mike Snitzer568c73a2019-01-18 14:10:37 -05001742 blk_qc_t ret = BLK_QC_T_NONE;
1743 struct dm_target *ti = md->immutable_target;
1744
1745 if (unlikely(!map)) {
1746 bio_io_error(bio);
1747 return ret;
1748 }
1749
1750 if (!ti) {
1751 ti = dm_table_find_target(map, bio->bi_iter.bi_sector);
Mikulas Patocka123d87d2019-08-23 09:55:26 -04001752 if (unlikely(!ti)) {
Mike Snitzer568c73a2019-01-18 14:10:37 -05001753 bio_io_error(bio);
1754 return ret;
1755 }
1756 }
1757
1758 /*
1759 * If in ->make_request_fn we need to use blk_queue_split(), otherwise
1760 * queue_limits for abnormal requests (e.g. discard, writesame, etc)
1761 * won't be imposed.
1762 */
1763 if (current->bio_list) {
Mike Snitzereffd58c2019-02-22 09:52:02 -05001764 blk_queue_split(md->queue, &bio);
1765 if (!is_abnormal_io(bio))
Mike Snitzer568c73a2019-01-18 14:10:37 -05001766 dm_queue_split(md, ti, &bio);
1767 }
1768
Mike Snitzer6548c7c2019-01-17 14:33:01 -05001769 if (dm_get_md_type(md) == DM_TYPE_NVME_BIO_BASED)
Mike Snitzer568c73a2019-01-18 14:10:37 -05001770 return __process_bio(md, map, bio, ti);
Mike Snitzer6548c7c2019-01-17 14:33:01 -05001771 else
1772 return __split_and_process_bio(md, map, bio);
1773}
1774
Mikulas Patocka24113d42018-11-06 22:34:59 +01001775static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001777 struct mapped_device *md = q->queuedata;
Mike Snitzer978e51b2017-12-09 15:16:42 -05001778 blk_qc_t ret = BLK_QC_T_NONE;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001779 int srcu_idx;
1780 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001782 map = dm_get_live_table(md, &srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783
Tejun Heo6a8736d2010-09-08 18:07:00 +02001784 /* if we're suspended, we have to queue this io for later */
1785 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001786 dm_put_live_table(md, srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001787
Jens Axboe1eff9d32016-08-05 15:35:16 -06001788 if (!(bio->bi_opf & REQ_RAHEAD))
Tejun Heo6a8736d2010-09-08 18:07:00 +02001789 queue_io(md, bio);
1790 else
Alasdair G Kergon54d9a1b2009-04-09 00:27:14 +01001791 bio_io_error(bio);
Mike Snitzer978e51b2017-12-09 15:16:42 -05001792 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793 }
1794
Mike Snitzer6548c7c2019-01-17 14:33:01 -05001795 ret = dm_process_bio(md, map, bio);
Mike Snitzer978e51b2017-12-09 15:16:42 -05001796
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001797 dm_put_live_table(md, srcu_idx);
Mike Snitzer978e51b2017-12-09 15:16:42 -05001798 return ret;
1799}
1800
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801static int dm_any_congested(void *congested_data, int bdi_bits)
1802{
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001803 int r = bdi_bits;
1804 struct mapped_device *md = congested_data;
1805 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001807 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Mike Snitzere522c032016-02-02 22:35:06 -05001808 if (dm_request_based(md)) {
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001809 /*
Mike Snitzere522c032016-02-02 22:35:06 -05001810 * With request-based DM we only need to check the
1811 * top-level queue for congestion.
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001812 */
Jan Karadc3b17c2017-02-02 15:56:50 +01001813 r = md->queue->backing_dev_info->wb.state & bdi_bits;
Mike Snitzere522c032016-02-02 22:35:06 -05001814 } else {
1815 map = dm_get_live_table_fast(md);
1816 if (map)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001817 r = dm_table_any_congested(map, bdi_bits);
Mike Snitzere522c032016-02-02 22:35:06 -05001818 dm_put_live_table_fast(md);
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001819 }
1820 }
1821
Linus Torvalds1da177e2005-04-16 15:20:36 -07001822 return r;
1823}
1824
1825/*-----------------------------------------------------------------
1826 * An IDR is used to keep track of allocated minor numbers.
1827 *---------------------------------------------------------------*/
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001828static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001830 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001832 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833}
1834
1835/*
1836 * See if the device with a specific minor # is free.
1837 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001838static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839{
Tejun Heoc9d76be2013-02-27 17:04:26 -08001840 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001841
1842 if (minor >= (1 << MINORBITS))
1843 return -EINVAL;
1844
Tejun Heoc9d76be2013-02-27 17:04:26 -08001845 idr_preload(GFP_KERNEL);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001846 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847
Tejun Heoc9d76be2013-02-27 17:04:26 -08001848 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001849
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001850 spin_unlock(&_minor_lock);
Tejun Heoc9d76be2013-02-27 17:04:26 -08001851 idr_preload_end();
1852 if (r < 0)
1853 return r == -ENOSPC ? -EBUSY : r;
1854 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855}
1856
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001857static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001858{
Tejun Heoc9d76be2013-02-27 17:04:26 -08001859 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860
Tejun Heoc9d76be2013-02-27 17:04:26 -08001861 idr_preload(GFP_KERNEL);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001862 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863
Tejun Heoc9d76be2013-02-27 17:04:26 -08001864 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001866 spin_unlock(&_minor_lock);
Tejun Heoc9d76be2013-02-27 17:04:26 -08001867 idr_preload_end();
1868 if (r < 0)
1869 return r;
1870 *minor = r;
1871 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872}
1873
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07001874static const struct block_device_operations dm_blk_dops;
Dan Williamsf26c5712017-04-12 12:35:44 -07001875static const struct dax_operations dm_dax_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876
Mikulas Patocka53d59142009-04-02 19:55:37 +01001877static void dm_wq_work(struct work_struct *work);
1878
Mike Snitzerc12c9a32018-01-12 09:32:21 -05001879static void dm_init_normal_md_queue(struct mapped_device *md)
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05001880{
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05001881 /*
1882 * Initialize aspects of queue that aren't relevant for blk-mq
1883 */
Jan Karadc3b17c2017-02-02 15:56:50 +01001884 md->queue->backing_dev_info->congested_fn = dm_any_congested;
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01001885}
1886
Mike Snitzer0f209722015-04-28 11:50:29 -04001887static void cleanup_mapped_device(struct mapped_device *md)
1888{
Mike Snitzer0f209722015-04-28 11:50:29 -04001889 if (md->wq)
1890 destroy_workqueue(md->wq);
Kent Overstreet6f1c8192018-05-20 18:25:53 -04001891 bioset_exit(&md->bs);
1892 bioset_exit(&md->io_bs);
Mike Snitzer0f209722015-04-28 11:50:29 -04001893
Dan Williamsf26c5712017-04-12 12:35:44 -07001894 if (md->dax_dev) {
1895 kill_dax(md->dax_dev);
1896 put_dax(md->dax_dev);
1897 md->dax_dev = NULL;
1898 }
1899
Mike Snitzer0f209722015-04-28 11:50:29 -04001900 if (md->disk) {
1901 spin_lock(&_minor_lock);
1902 md->disk->private_data = NULL;
1903 spin_unlock(&_minor_lock);
Mike Snitzer0f209722015-04-28 11:50:29 -04001904 del_gendisk(md->disk);
1905 put_disk(md->disk);
1906 }
1907
1908 if (md->queue)
1909 blk_cleanup_queue(md->queue);
1910
Tahsin Erdogand09960b2016-10-10 05:35:19 -07001911 cleanup_srcu_struct(&md->io_barrier);
1912
Mike Snitzer0f209722015-04-28 11:50:29 -04001913 if (md->bdev) {
1914 bdput(md->bdev);
1915 md->bdev = NULL;
1916 }
Mike Snitzer4cc96132016-05-12 16:28:10 -04001917
Mike Snitzerd5ffebd2018-01-05 21:17:20 -05001918 mutex_destroy(&md->suspend_lock);
1919 mutex_destroy(&md->type_lock);
1920 mutex_destroy(&md->table_devices_lock);
1921
Mike Snitzer4cc96132016-05-12 16:28:10 -04001922 dm_mq_cleanup_mapped_device(md);
Mike Snitzer0f209722015-04-28 11:50:29 -04001923}
1924
Linus Torvalds1da177e2005-04-16 15:20:36 -07001925/*
1926 * Allocate and initialise a blank device with a given minor.
1927 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001928static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001929{
Mike Snitzer115485e2016-02-22 12:16:21 -05001930 int r, numa_node_id = dm_get_numa_node();
1931 struct mapped_device *md;
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001932 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001933
Mikulas Patocka856eb092017-10-31 19:33:02 -04001934 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001935 if (!md) {
1936 DMWARN("unable to allocate device, out of memory.");
1937 return NULL;
1938 }
1939
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001940 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001941 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001942
Linus Torvalds1da177e2005-04-16 15:20:36 -07001943 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001944 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001945 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001946 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001947 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001949 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001951 r = init_srcu_struct(&md->io_barrier);
1952 if (r < 0)
1953 goto bad_io_barrier;
1954
Mike Snitzer115485e2016-02-22 12:16:21 -05001955 md->numa_node_id = numa_node_id;
Mike Snitzer591ddcf2016-01-31 12:05:42 -05001956 md->init_tio_pdu = false;
Mike Snitzera5664da2010-08-12 04:14:01 +01001957 md->type = DM_TYPE_NONE;
Daniel Walkere61290a2008-02-08 02:10:08 +00001958 mutex_init(&md->suspend_lock);
Mike Snitzera5664da2010-08-12 04:14:01 +01001959 mutex_init(&md->type_lock);
Benjamin Marzinski86f11522014-08-13 13:53:43 -05001960 mutex_init(&md->table_devices_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001961 spin_lock_init(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001962 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001963 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001965 atomic_set(&md->uevent_seq, 0);
1966 INIT_LIST_HEAD(&md->uevent_list);
Benjamin Marzinski86f11522014-08-13 13:53:43 -05001967 INIT_LIST_HEAD(&md->table_devices);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001968 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969
Christoph Hellwig6d469642018-11-14 17:02:18 +01001970 md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971 if (!md->queue)
Mike Snitzer0f209722015-04-28 11:50:29 -04001972 goto bad;
Mike Snitzerc12c9a32018-01-12 09:32:21 -05001973 md->queue->queuedata = md;
1974 md->queue->backing_dev_info->congested_data = md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001975
Mike Snitzerc12c9a32018-01-12 09:32:21 -05001976 md->disk = alloc_disk_node(1, md->numa_node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977 if (!md->disk)
Mike Snitzer0f209722015-04-28 11:50:29 -04001978 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001980 init_waitqueue_head(&md->wait);
Mikulas Patocka53d59142009-04-02 19:55:37 +01001981 INIT_WORK(&md->work, dm_wq_work);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001982 init_waitqueue_head(&md->eventq);
Mikulas Patocka2995fa72014-01-13 19:37:54 -05001983 init_completion(&md->kobj_holder.completion);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001984
Linus Torvalds1da177e2005-04-16 15:20:36 -07001985 md->disk->major = _major;
1986 md->disk->first_minor = minor;
1987 md->disk->fops = &dm_blk_dops;
1988 md->disk->queue = md->queue;
1989 md->disk->private_data = md;
1990 sprintf(md->disk->disk_name, "dm-%d", minor);
Dan Williamsf26c5712017-04-12 12:35:44 -07001991
Dan Williams976431b2018-03-29 17:22:13 -07001992 if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
Pankaj Guptafefc1d92019-07-05 19:33:24 +05301993 md->dax_dev = alloc_dax(md, md->disk->disk_name,
1994 &dm_dax_ops, 0);
Peng Wang514cf4f2019-04-18 21:59:19 +08001995 if (!md->dax_dev)
Dan Williams976431b2018-03-29 17:22:13 -07001996 goto bad;
1997 }
Dan Williamsf26c5712017-04-12 12:35:44 -07001998
Mike Snitzerc100ec42018-01-08 20:03:04 -05001999 add_disk_no_queue_reg(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08002000 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002001
Tejun Heo670368a2013-07-30 08:40:21 -04002002 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
Milan Broz304f3f62008-02-08 02:11:17 +00002003 if (!md->wq)
Mike Snitzer0f209722015-04-28 11:50:29 -04002004 goto bad;
Milan Broz304f3f62008-02-08 02:11:17 +00002005
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002006 md->bdev = bdget_disk(md->disk, 0);
2007 if (!md->bdev)
Mike Snitzer0f209722015-04-28 11:50:29 -04002008 goto bad;
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002009
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002010 dm_stats_init(&md->stats);
2011
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07002012 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002013 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07002014 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002015 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07002016
2017 BUG_ON(old_md != MINOR_ALLOCED);
2018
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 return md;
2020
Mike Snitzer0f209722015-04-28 11:50:29 -04002021bad:
2022 cleanup_mapped_device(md);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002023bad_io_barrier:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002024 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00002025bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07002026 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00002027bad_module_get:
Mikulas Patocka856eb092017-10-31 19:33:02 -04002028 kvfree(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029 return NULL;
2030}
2031
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01002032static void unlock_fs(struct mapped_device *md);
2033
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034static void free_dev(struct mapped_device *md)
2035{
Tejun Heof331c022008-09-03 09:01:48 +02002036 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08002037
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002038 unlock_fs(md);
Keith Busch2eb6e1e2014-10-17 17:46:36 -06002039
Mike Snitzer0f209722015-04-28 11:50:29 -04002040 cleanup_mapped_device(md);
Mike Snitzer0f209722015-04-28 11:50:29 -04002041
2042 free_table_devices(&md->table_devices);
2043 dm_stats_cleanup(&md->stats);
Mike Snitzer63a4f062015-03-23 17:01:43 -04002044 free_minor(minor);
2045
Jeff Mahoney10da4f72006-06-26 00:27:25 -07002046 module_put(THIS_MODULE);
Mikulas Patocka856eb092017-10-31 19:33:02 -04002047 kvfree(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048}
2049
Jens Axboe2a2a4c52018-06-07 14:42:06 -06002050static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002051{
Mikulas Patockac0820cf2012-12-21 20:23:38 +00002052 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
Jens Axboe2a2a4c52018-06-07 14:42:06 -06002053 int ret = 0;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002054
Mike Snitzer0776aa02017-12-08 14:40:52 -05002055 if (dm_table_bio_based(t)) {
Mike Snitzer64f52b02017-12-11 23:17:47 -05002056 /*
2057 * The md may already have mempools that need changing.
2058 * If so, reload bioset because front_pad may have changed
2059 * because a different table was loaded.
2060 */
Kent Overstreet6f1c8192018-05-20 18:25:53 -04002061 bioset_exit(&md->bs);
2062 bioset_exit(&md->io_bs);
Mike Snitzer0776aa02017-12-08 14:40:52 -05002063
Kent Overstreet6f1c8192018-05-20 18:25:53 -04002064 } else if (bioset_initialized(&md->bs)) {
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04002065 /*
2066 * There's no need to reload with request-based dm
2067 * because the size of front_pad doesn't change.
2068 * Note for future: If you are to reload bioset,
2069 * prep-ed requests in the queue may refer
2070 * to bio from the old bioset, so you must walk
2071 * through the queue to unprep.
2072 */
2073 goto out;
Mikulas Patockac0820cf2012-12-21 20:23:38 +00002074 }
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002075
Kent Overstreet6f1c8192018-05-20 18:25:53 -04002076 BUG_ON(!p ||
2077 bioset_initialized(&md->bs) ||
2078 bioset_initialized(&md->io_bs));
Mike Snitzercbc4e3c2015-04-27 16:37:50 -04002079
Jens Axboe2a2a4c52018-06-07 14:42:06 -06002080 ret = bioset_init_from_src(&md->bs, &p->bs);
2081 if (ret)
2082 goto out;
2083 ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
2084 if (ret)
2085 bioset_exit(&md->bs);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002086out:
Mike Snitzer02233342015-03-10 23:49:26 -04002087 /* mempool bind completed, no longer need any mempools in the table */
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002088 dm_table_free_md_mempools(t);
Jens Axboe2a2a4c52018-06-07 14:42:06 -06002089 return ret;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002090}
2091
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092/*
2093 * Bind a table to the device.
2094 */
2095static void event_callback(void *context)
2096{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002097 unsigned long flags;
2098 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002099 struct mapped_device *md = (struct mapped_device *) context;
2100
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002101 spin_lock_irqsave(&md->uevent_lock, flags);
2102 list_splice_init(&md->uevent_list, &uevents);
2103 spin_unlock_irqrestore(&md->uevent_lock, flags);
2104
Tejun Heoed9e1982008-08-25 19:56:05 +09002105 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002106
Linus Torvalds1da177e2005-04-16 15:20:36 -07002107 atomic_inc(&md->event_nr);
2108 wake_up(&md->eventq);
Mikulas Patocka62e08242017-09-20 07:29:49 -04002109 dm_issue_global_event();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002110}
2111
Mike Snitzerc2176492011-01-13 19:53:46 +00002112/*
2113 * Protected by md->suspend_lock obtained by dm_swap_table().
2114 */
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07002115static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116{
Bart Van Assche1ea06542017-04-27 10:11:21 -07002117 lockdep_assert_held(&md->suspend_lock);
2118
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07002119 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002120
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002121 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002122}
2123
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002124/*
2125 * Returns old map, which caller must destroy.
2126 */
2127static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2128 struct queue_limits *limits)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002129{
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002130 struct dm_table *old_map;
Jens Axboe165125e2007-07-24 09:28:11 +02002131 struct request_queue *q = md->queue;
Mike Snitzer978e51b2017-12-09 15:16:42 -05002132 bool request_based = dm_table_request_based(t);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002133 sector_t size;
Jens Axboe2a2a4c52018-06-07 14:42:06 -06002134 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002135
Bart Van Assche5a8f1f82016-08-31 15:17:04 -07002136 lockdep_assert_held(&md->suspend_lock);
2137
Linus Torvalds1da177e2005-04-16 15:20:36 -07002138 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08002139
2140 /*
2141 * Wipe any geometry if the size of the table changed.
2142 */
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002143 if (size != dm_get_size(md))
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08002144 memset(&md->geometry, 0, sizeof(md->geometry));
2145
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002146 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002147
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002148 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002149
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002150 /*
2151 * The queue hasn't been stopped yet, if the old table type wasn't
2152 * for request-based during suspension. So stop it to prevent
2153 * I/O mapping before resume.
2154 * This must be done before setting the queue restrictions,
2155 * because request-based dm may be run just after the setting.
2156 */
Mike Snitzer978e51b2017-12-09 15:16:42 -05002157 if (request_based)
Mike Snitzereca7ee62016-02-20 13:45:38 -05002158 dm_stop_queue(q);
Mike Snitzer978e51b2017-12-09 15:16:42 -05002159
2160 if (request_based || md->type == DM_TYPE_NVME_BIO_BASED) {
Mike Snitzer16f12262016-01-31 17:22:27 -05002161 /*
Mike Snitzer978e51b2017-12-09 15:16:42 -05002162 * Leverage the fact that request-based DM targets and
2163 * NVMe bio based targets are immutable singletons
2164 * - used to optimize both dm_request_fn and dm_mq_queue_rq;
2165 * and __process_bio.
Mike Snitzer16f12262016-01-31 17:22:27 -05002166 */
2167 md->immutable_target = dm_table_get_immutable_target(t);
2168 }
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002169
Jens Axboe2a2a4c52018-06-07 14:42:06 -06002170 ret = __bind_mempools(md, t);
2171 if (ret) {
2172 old_map = ERR_PTR(ret);
2173 goto out;
2174 }
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002175
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002176 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzer1d3aa6f2016-02-22 14:14:24 -05002177 rcu_assign_pointer(md->map, (void *)t);
Alasdair G Kergon36a04562011-10-31 20:19:04 +00002178 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2179
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002180 dm_table_set_restrictions(t, q, limits);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002181 if (old_map)
2182 dm_sync_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002183
Jens Axboe2a2a4c52018-06-07 14:42:06 -06002184out:
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002185 return old_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002186}
2187
Alasdair G Kergona7940152009-12-10 23:52:23 +00002188/*
2189 * Returns unbound table for the caller to free.
2190 */
2191static struct dm_table *__unbind(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002192{
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002193 struct dm_table *map = rcu_dereference_protected(md->map, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002194
2195 if (!map)
Alasdair G Kergona7940152009-12-10 23:52:23 +00002196 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002197
2198 dm_table_event_callback(map, NULL, NULL);
Monam Agarwal9cdb8522014-03-23 23:58:27 +05302199 RCU_INIT_POINTER(md->map, NULL);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002200 dm_sync_table(md);
Alasdair G Kergona7940152009-12-10 23:52:23 +00002201
2202 return map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203}
2204
2205/*
2206 * Constructor for a new device.
2207 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07002208int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002209{
Mike Snitzerc12c9a32018-01-12 09:32:21 -05002210 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002211 struct mapped_device *md;
2212
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07002213 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214 if (!md)
2215 return -ENXIO;
2216
Mike Snitzerc12c9a32018-01-12 09:32:21 -05002217 r = dm_sysfs_init(md);
2218 if (r) {
2219 free_dev(md);
2220 return r;
2221 }
Milan Broz784aae72009-01-06 03:05:12 +00002222
Linus Torvalds1da177e2005-04-16 15:20:36 -07002223 *result = md;
2224 return 0;
2225}
2226
Mike Snitzera5664da2010-08-12 04:14:01 +01002227/*
2228 * Functions to manage md->type.
2229 * All are required to hold md->type_lock.
2230 */
2231void dm_lock_md_type(struct mapped_device *md)
2232{
2233 mutex_lock(&md->type_lock);
2234}
2235
2236void dm_unlock_md_type(struct mapped_device *md)
2237{
2238 mutex_unlock(&md->type_lock);
2239}
2240
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002241void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
Mike Snitzera5664da2010-08-12 04:14:01 +01002242{
Mike Snitzer00c4fc32013-08-27 18:57:03 -04002243 BUG_ON(!mutex_is_locked(&md->type_lock));
Mike Snitzera5664da2010-08-12 04:14:01 +01002244 md->type = type;
2245}
2246
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002247enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
Mike Snitzera5664da2010-08-12 04:14:01 +01002248{
2249 return md->type;
2250}
2251
Alasdair G Kergon36a04562011-10-31 20:19:04 +00002252struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2253{
2254 return md->immutable_target_type;
2255}
2256
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002257/*
Mike Snitzerf84cb8a2013-09-19 12:13:58 -04002258 * The queue_limits are only valid as long as you have a reference
2259 * count on 'md'.
2260 */
2261struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2262{
2263 BUG_ON(!atomic_read(&md->holders));
2264 return &md->queue->limits;
2265}
2266EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2267
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002268/*
2269 * Setup the DM device's queue based on md's type
2270 */
Mike Snitzer591ddcf2016-01-31 12:05:42 -05002271int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002272{
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002273 int r;
Mike Snitzerc100ec42018-01-08 20:03:04 -05002274 struct queue_limits limits;
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002275 enum dm_queue_mode type = dm_get_md_type(md);
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002276
Toshi Kani545ed202016-06-22 17:54:53 -06002277 switch (type) {
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002278 case DM_TYPE_REQUEST_BASED:
Mike Snitzere83068a2016-05-24 21:16:51 -04002279 r = dm_mq_init_request_queue(md, t);
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002280 if (r) {
Mike Snitzereca7ee62016-02-20 13:45:38 -05002281 DMERR("Cannot initialize queue for request-based dm-mq mapped device");
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002282 return r;
2283 }
2284 break;
2285 case DM_TYPE_BIO_BASED:
Toshi Kani545ed202016-06-22 17:54:53 -06002286 case DM_TYPE_DAX_BIO_BASED:
Mike Snitzer978e51b2017-12-09 15:16:42 -05002287 case DM_TYPE_NVME_BIO_BASED:
2288 dm_init_normal_md_queue(md);
Mikulas Patocka24113d42018-11-06 22:34:59 +01002289 blk_queue_make_request(md->queue, dm_make_request);
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002290 break;
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002291 case DM_TYPE_NONE:
2292 WARN_ON_ONCE(true);
2293 break;
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002294 }
2295
Mike Snitzerc100ec42018-01-08 20:03:04 -05002296 r = dm_calculate_queue_limits(t, &limits);
2297 if (r) {
2298 DMERR("Cannot calculate initial queue limits");
2299 return r;
2300 }
2301 dm_table_set_restrictions(t, md->queue, &limits);
2302 blk_register_queue(md->disk);
2303
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002304 return 0;
2305}
2306
Mikulas Patocka2bec1f42015-02-17 14:30:53 -05002307struct mapped_device *dm_get_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002308{
2309 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002310 unsigned minor = MINOR(dev);
2311
2312 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2313 return NULL;
2314
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002315 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002316
2317 md = idr_find(&_minor_idr, minor);
Mike Snitzer49de5762017-11-06 16:40:10 -05002318 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2319 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2320 md = NULL;
2321 goto out;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002322 }
Mike Snitzer49de5762017-11-06 16:40:10 -05002323 dm_get(md);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002324out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002325 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002326
David Teigland637842c2006-01-06 00:20:00 -08002327 return md;
2328}
Alasdair G Kergon3cf2e4b2011-10-31 20:19:06 +00002329EXPORT_SYMBOL_GPL(dm_get_md);
David Teiglandd229a952006-01-06 00:20:01 -08002330
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002331void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08002332{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002333 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002334}
2335
2336void dm_set_mdptr(struct mapped_device *md, void *ptr)
2337{
2338 md->interface_ptr = ptr;
2339}
2340
2341void dm_get(struct mapped_device *md)
2342{
2343 atomic_inc(&md->holders);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002344 BUG_ON(test_bit(DMF_FREEING, &md->flags));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002345}
2346
Mikulas Patocka09ee96b2015-02-26 11:41:28 -05002347int dm_hold(struct mapped_device *md)
2348{
2349 spin_lock(&_minor_lock);
2350 if (test_bit(DMF_FREEING, &md->flags)) {
2351 spin_unlock(&_minor_lock);
2352 return -EBUSY;
2353 }
2354 dm_get(md);
2355 spin_unlock(&_minor_lock);
2356 return 0;
2357}
2358EXPORT_SYMBOL_GPL(dm_hold);
2359
Alasdair G Kergon72d94862006-06-26 00:27:35 -07002360const char *dm_device_name(struct mapped_device *md)
2361{
2362 return md->name;
2363}
2364EXPORT_SYMBOL_GPL(dm_device_name);
2365
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002366static void __dm_destroy(struct mapped_device *md, bool wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002367{
Mike Anderson1134e5a2006-03-27 01:17:54 -08002368 struct dm_table *map;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002369 int srcu_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002371 might_sleep();
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002372
Mike Snitzer63a4f062015-03-23 17:01:43 -04002373 spin_lock(&_minor_lock);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002374 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2375 set_bit(DMF_FREEING, &md->flags);
2376 spin_unlock(&_minor_lock);
2377
Mike Snitzerc12c9a32018-01-12 09:32:21 -05002378 blk_set_queue_dying(md->queue);
Bart Van Assche3b785fb2016-08-31 15:17:49 -07002379
Mikulas Patockaab7c7bb2015-02-27 14:04:27 -05002380 /*
2381 * Take suspend_lock so that presuspend and postsuspend methods
2382 * do not race with internal suspend.
2383 */
2384 mutex_lock(&md->suspend_lock);
Junichi Nomura2a708cf2015-10-01 08:31:51 +00002385 map = dm_get_live_table(md, &srcu_idx);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002386 if (!dm_suspended_md(md)) {
2387 dm_table_presuspend_targets(map);
2388 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389 }
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002390 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2391 dm_put_live_table(md, srcu_idx);
Junichi Nomura2a708cf2015-10-01 08:31:51 +00002392 mutex_unlock(&md->suspend_lock);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002393
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002394 /*
2395 * Rare, but there may be I/O requests still going to complete,
2396 * for example. Wait for all references to disappear.
2397 * No one should increment the reference count of the mapped_device,
2398 * after the mapped_device state becomes DMF_FREEING.
2399 */
2400 if (wait)
2401 while (atomic_read(&md->holders))
2402 msleep(1);
2403 else if (atomic_read(&md->holders))
2404 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2405 dm_device_name(md), atomic_read(&md->holders));
2406
2407 dm_sysfs_exit(md);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002408 dm_table_destroy(__unbind(md));
2409 free_dev(md);
2410}
2411
2412void dm_destroy(struct mapped_device *md)
2413{
2414 __dm_destroy(md, true);
2415}
2416
2417void dm_destroy_immediate(struct mapped_device *md)
2418{
2419 __dm_destroy(md, false);
2420}
2421
2422void dm_put(struct mapped_device *md)
2423{
2424 atomic_dec(&md->holders);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425}
Edward Goggin79eb8852007-05-09 02:32:56 -07002426EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002427
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002428static int dm_wait_for_completion(struct mapped_device *md, long task_state)
Milan Broz46125c12008-02-08 02:10:30 +00002429{
2430 int r = 0;
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07002431 DEFINE_WAIT(wait);
Milan Broz46125c12008-02-08 02:10:30 +00002432
2433 while (1) {
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07002434 prepare_to_wait(&md->wait, &wait, task_state);
Milan Broz46125c12008-02-08 02:10:30 +00002435
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00002436 if (!md_in_flight(md))
Milan Broz46125c12008-02-08 02:10:30 +00002437 break;
2438
Bart Van Asschee3fabdf2016-08-31 15:16:22 -07002439 if (signal_pending_state(task_state, current)) {
Milan Broz46125c12008-02-08 02:10:30 +00002440 r = -EINTR;
2441 break;
2442 }
2443
2444 io_schedule();
2445 }
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07002446 finish_wait(&md->wait, &wait);
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002447
Milan Broz46125c12008-02-08 02:10:30 +00002448 return r;
2449}
2450
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451/*
2452 * Process the deferred bios
2453 */
Mikulas Patockaef208582009-04-02 19:55:38 +01002454static void dm_wq_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002455{
Mikulas Patockaef208582009-04-02 19:55:38 +01002456 struct mapped_device *md = container_of(work, struct mapped_device,
2457 work);
Milan Broz6d6f10d2008-02-08 02:10:22 +00002458 struct bio *c;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002459 int srcu_idx;
2460 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002462 map = dm_get_live_table(md, &srcu_idx);
Mikulas Patockaef208582009-04-02 19:55:38 +01002463
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002464 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002465 spin_lock_irq(&md->deferred_lock);
2466 c = bio_list_pop(&md->deferred);
2467 spin_unlock_irq(&md->deferred_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002468
Tejun Heo6a8736d2010-09-08 18:07:00 +02002469 if (!c)
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002470 break;
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002471
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002472 if (dm_request_based(md))
Mike Snitzer6548c7c2019-01-17 14:33:01 -05002473 (void) generic_make_request(c);
Tejun Heo6a8736d2010-09-08 18:07:00 +02002474 else
Mike Snitzer6548c7c2019-01-17 14:33:01 -05002475 (void) dm_process_bio(md, map, c);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002476 }
Milan Broz73d410c2008-02-08 02:10:25 +00002477
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002478 dm_put_live_table(md, srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002479}
2480
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002481static void dm_queue_flush(struct mapped_device *md)
Milan Broz304f3f62008-02-08 02:11:17 +00002482{
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002483 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01002484 smp_mb__after_atomic();
Mikulas Patocka53d59142009-04-02 19:55:37 +01002485 queue_work(md->wq, &md->work);
Milan Broz304f3f62008-02-08 02:11:17 +00002486}
2487
Linus Torvalds1da177e2005-04-16 15:20:36 -07002488/*
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002489 * Swap in a new table, returning the old one for the caller to destroy.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490 */
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002491struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492{
Mike Christie87eb5b22013-03-01 22:45:48 +00002493 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002494 struct queue_limits limits;
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002495 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002496
Daniel Walkere61290a2008-02-08 02:10:08 +00002497 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002498
2499 /* device must be suspended */
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +00002500 if (!dm_suspended_md(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002501 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002502
Mike Snitzer3ae70652012-09-26 23:45:45 +01002503 /*
2504 * If the new table has no data devices, retain the existing limits.
2505 * This helps multipath with queue_if_no_path if all paths disappear,
2506 * then new I/O is queued based on these limits, and then some paths
2507 * reappear.
2508 */
2509 if (dm_table_has_no_data_devices(table)) {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002510 live_map = dm_get_live_table_fast(md);
Mike Snitzer3ae70652012-09-26 23:45:45 +01002511 if (live_map)
2512 limits = md->queue->limits;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002513 dm_put_live_table_fast(md);
Mike Snitzer3ae70652012-09-26 23:45:45 +01002514 }
2515
Mike Christie87eb5b22013-03-01 22:45:48 +00002516 if (!live_map) {
2517 r = dm_calculate_queue_limits(table, &limits);
2518 if (r) {
2519 map = ERR_PTR(r);
2520 goto out;
2521 }
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002522 }
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002523
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002524 map = __bind(md, table, &limits);
Mikulas Patocka62e08242017-09-20 07:29:49 -04002525 dm_issue_global_event();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002526
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002527out:
Daniel Walkere61290a2008-02-08 02:10:08 +00002528 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002529 return map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002530}
2531
2532/*
2533 * Functions to lock and unlock any filesystem running on the
2534 * device.
2535 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002536static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002537{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002538 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002539
2540 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002541
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002542 md->frozen_sb = freeze_bdev(md->bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002543 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002544 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002545 md->frozen_sb = NULL;
2546 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002547 }
2548
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002549 set_bit(DMF_FROZEN, &md->flags);
2550
Linus Torvalds1da177e2005-04-16 15:20:36 -07002551 return 0;
2552}
2553
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002554static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002555{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002556 if (!test_bit(DMF_FROZEN, &md->flags))
2557 return;
2558
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002559 thaw_bdev(md->bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002560 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002561 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562}
2563
2564/*
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002565 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2566 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2567 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2568 *
Mike Snitzerffcc3932014-10-28 18:34:52 -04002569 * If __dm_suspend returns 0, the device is completely quiescent
2570 * now. There is no request-processing activity. All new requests
2571 * are being added to md->deferred list.
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002572 */
Mike Snitzerffcc3932014-10-28 18:34:52 -04002573static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002574 unsigned suspend_flags, long task_state,
Mike Snitzereaf9a732016-08-02 13:07:20 -04002575 int dmf_suspended_flag)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002576{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002577 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2578 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2579 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002580
Bart Van Assche5a8f1f82016-08-31 15:17:04 -07002581 lockdep_assert_held(&md->suspend_lock);
2582
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002583 /*
2584 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2585 * This flag is cleared before dm_suspend returns.
2586 */
2587 if (noflush)
2588 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Bart Van Assche86331f32017-04-27 10:11:26 -07002589 else
2590 pr_debug("%s: suspending with flush\n", dm_device_name(md));
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002591
Mike Snitzerd67ee212014-10-28 20:13:31 -04002592 /*
2593 * This gets reverted if there's an error later and the targets
2594 * provide the .presuspend_undo hook.
2595 */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002596 dm_table_presuspend_targets(map);
2597
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002598 /*
Kiyoshi Ueda9f518b22009-12-10 23:52:16 +00002599 * Flush I/O to the device.
2600 * Any I/O submitted after lock_fs() may not be flushed.
2601 * noflush takes precedence over do_lockfs.
2602 * (lock_fs() flushes I/Os and waits for them to complete.)
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002603 */
2604 if (!noflush && do_lockfs) {
2605 r = lock_fs(md);
Mike Snitzerd67ee212014-10-28 20:13:31 -04002606 if (r) {
2607 dm_table_presuspend_undo_targets(map);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002608 return r;
Mike Snitzerd67ee212014-10-28 20:13:31 -04002609 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002610 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002611
2612 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002613 * Here we must make sure that no processes are submitting requests
2614 * to target drivers i.e. no one may be executing
2615 * __split_and_process_bio. This is called from dm_request and
2616 * dm_wq_work.
2617 *
2618 * To get all processes out of __split_and_process_bio in dm_request,
2619 * we take the write lock. To prevent any process from reentering
Tejun Heo6a8736d2010-09-08 18:07:00 +02002620 * __split_and_process_bio from dm_request and quiesce the thread
2621 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2622 * flush_workqueue(md->wq).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002623 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01002624 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002625 if (map)
2626 synchronize_srcu(&md->io_barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002627
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002628 /*
Tejun Heo29e40132010-09-08 18:07:00 +02002629 * Stop md->queue before flushing md->wq in case request-based
2630 * dm defers requests to md->wq from md->queue.
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002631 */
Jens Axboe6a23e052018-10-10 20:49:26 -06002632 if (dm_request_based(md))
Mike Snitzereca7ee62016-02-20 13:45:38 -05002633 dm_stop_queue(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002634
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002635 flush_workqueue(md->wq);
2636
Linus Torvalds1da177e2005-04-16 15:20:36 -07002637 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002638 * At this point no more requests are entering target request routines.
2639 * We call dm_wait_for_completion to wait for all existing requests
2640 * to finish.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002641 */
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002642 r = dm_wait_for_completion(md, task_state);
Mike Snitzereaf9a732016-08-02 13:07:20 -04002643 if (!r)
2644 set_bit(dmf_suspended_flag, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002645
Milan Broz6d6f10d2008-02-08 02:10:22 +00002646 if (noflush)
Mikulas Patocka022c2612009-04-02 19:55:39 +01002647 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002648 if (map)
2649 synchronize_srcu(&md->io_barrier);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002650
Linus Torvalds1da177e2005-04-16 15:20:36 -07002651 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00002652 if (r < 0) {
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002653 dm_queue_flush(md);
Milan Broz73d410c2008-02-08 02:10:25 +00002654
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002655 if (dm_request_based(md))
Mike Snitzereca7ee62016-02-20 13:45:38 -05002656 dm_start_queue(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002657
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002658 unlock_fs(md);
Mike Snitzerd67ee212014-10-28 20:13:31 -04002659 dm_table_presuspend_undo_targets(map);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002660 /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002661 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002662
Mike Snitzerffcc3932014-10-28 18:34:52 -04002663 return r;
2664}
2665
2666/*
2667 * We need to be able to change a mapping table under a mounted
2668 * filesystem. For example we might want to move some data in
2669 * the background. Before the table can be swapped with
2670 * dm_bind_table, dm_suspend must be called to flush any in
2671 * flight bios and ensure that any further io gets deferred.
2672 */
2673/*
2674 * Suspend mechanism in request-based dm.
2675 *
2676 * 1. Flush all I/Os by lock_fs() if needed.
2677 * 2. Stop dispatching any I/O by stopping the request_queue.
2678 * 3. Wait for all in-flight I/Os to be completed or requeued.
2679 *
2680 * To abort suspend, start the request_queue.
2681 */
2682int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2683{
2684 struct dm_table *map = NULL;
2685 int r = 0;
2686
2687retry:
2688 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2689
2690 if (dm_suspended_md(md)) {
2691 r = -EINVAL;
2692 goto out_unlock;
2693 }
2694
2695 if (dm_suspended_internally_md(md)) {
2696 /* already internally suspended, wait for internal resume */
2697 mutex_unlock(&md->suspend_lock);
2698 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2699 if (r)
2700 return r;
2701 goto retry;
2702 }
2703
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002704 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002705
Mike Snitzereaf9a732016-08-02 13:07:20 -04002706 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002707 if (r)
2708 goto out_unlock;
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002709
Kiyoshi Ueda4d4471c2009-12-10 23:52:26 +00002710 dm_table_postsuspend_targets(map);
2711
Alasdair G Kergond2874832006-11-08 17:44:43 -08002712out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00002713 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002714 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002715}
2716
Mike Snitzerffcc3932014-10-28 18:34:52 -04002717static int __dm_resume(struct mapped_device *md, struct dm_table *map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002719 if (map) {
2720 int r = dm_table_resume_targets(map);
2721 if (r)
2722 return r;
2723 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002724
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002725 dm_queue_flush(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002726
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002727 /*
2728 * Flushing deferred I/Os must be done after targets are resumed
2729 * so that mapping of targets can work correctly.
2730 * Request-based dm is queueing the deferred I/Os in its request_queue.
2731 */
2732 if (dm_request_based(md))
Mike Snitzereca7ee62016-02-20 13:45:38 -05002733 dm_start_queue(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002734
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002735 unlock_fs(md);
2736
Mike Snitzerffcc3932014-10-28 18:34:52 -04002737 return 0;
2738}
2739
2740int dm_resume(struct mapped_device *md)
2741{
Minfei Huang8dc23652016-09-06 16:00:29 +08002742 int r;
Mike Snitzerffcc3932014-10-28 18:34:52 -04002743 struct dm_table *map = NULL;
2744
2745retry:
Minfei Huang8dc23652016-09-06 16:00:29 +08002746 r = -EINVAL;
Mike Snitzerffcc3932014-10-28 18:34:52 -04002747 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2748
2749 if (!dm_suspended_md(md))
2750 goto out;
2751
2752 if (dm_suspended_internally_md(md)) {
2753 /* already internally suspended, wait for internal resume */
2754 mutex_unlock(&md->suspend_lock);
2755 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2756 if (r)
2757 return r;
2758 goto retry;
2759 }
2760
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002761 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002762 if (!map || !dm_table_get_size(map))
2763 goto out;
2764
2765 r = __dm_resume(md, map);
2766 if (r)
2767 goto out;
2768
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002769 clear_bit(DMF_SUSPENDED, &md->flags);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002770out:
Daniel Walkere61290a2008-02-08 02:10:08 +00002771 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002772
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002773 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002774}
2775
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002776/*
2777 * Internal suspend/resume works like userspace-driven suspend. It waits
2778 * until all bios finish and prevents issuing new bios to the target drivers.
2779 * It may be used only from the kernel.
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002780 */
2781
Mike Snitzerffcc3932014-10-28 18:34:52 -04002782static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2783{
2784 struct dm_table *map = NULL;
2785
Bart Van Assche1ea06542017-04-27 10:11:21 -07002786 lockdep_assert_held(&md->suspend_lock);
2787
Mikulas Patocka96b26c82015-01-08 18:52:26 -05002788 if (md->internal_suspend_count++)
Mike Snitzerffcc3932014-10-28 18:34:52 -04002789 return; /* nested internal suspend */
2790
2791 if (dm_suspended_md(md)) {
2792 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2793 return; /* nest suspend */
2794 }
2795
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002796 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002797
2798 /*
2799 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2800 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2801 * would require changing .presuspend to return an error -- avoid this
2802 * until there is a need for more elaborate variants of internal suspend.
2803 */
Mike Snitzereaf9a732016-08-02 13:07:20 -04002804 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2805 DMF_SUSPENDED_INTERNALLY);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002806
2807 dm_table_postsuspend_targets(map);
2808}
2809
2810static void __dm_internal_resume(struct mapped_device *md)
2811{
Mikulas Patocka96b26c82015-01-08 18:52:26 -05002812 BUG_ON(!md->internal_suspend_count);
2813
2814 if (--md->internal_suspend_count)
Mike Snitzerffcc3932014-10-28 18:34:52 -04002815 return; /* resume from nested internal suspend */
2816
2817 if (dm_suspended_md(md))
2818 goto done; /* resume from nested suspend */
2819
2820 /*
2821 * NOTE: existing callers don't need to call dm_table_resume_targets
2822 * (which may fail -- so best to avoid it for now by passing NULL map)
2823 */
2824 (void) __dm_resume(md, NULL);
2825
2826done:
2827 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2828 smp_mb__after_atomic();
2829 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2830}
2831
2832void dm_internal_suspend_noflush(struct mapped_device *md)
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002833{
2834 mutex_lock(&md->suspend_lock);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002835 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2836 mutex_unlock(&md->suspend_lock);
2837}
2838EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2839
2840void dm_internal_resume(struct mapped_device *md)
2841{
2842 mutex_lock(&md->suspend_lock);
2843 __dm_internal_resume(md);
2844 mutex_unlock(&md->suspend_lock);
2845}
2846EXPORT_SYMBOL_GPL(dm_internal_resume);
2847
2848/*
2849 * Fast variants of internal suspend/resume hold md->suspend_lock,
2850 * which prevents interaction with userspace-driven suspend.
2851 */
2852
2853void dm_internal_suspend_fast(struct mapped_device *md)
2854{
2855 mutex_lock(&md->suspend_lock);
2856 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002857 return;
2858
2859 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2860 synchronize_srcu(&md->io_barrier);
2861 flush_workqueue(md->wq);
2862 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2863}
Mikulas Patockab735fed2015-02-26 11:40:35 -05002864EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002865
Mike Snitzerffcc3932014-10-28 18:34:52 -04002866void dm_internal_resume_fast(struct mapped_device *md)
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002867{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002868 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002869 goto done;
2870
2871 dm_queue_flush(md);
2872
2873done:
2874 mutex_unlock(&md->suspend_lock);
2875}
Mikulas Patockab735fed2015-02-26 11:40:35 -05002876EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002877
Linus Torvalds1da177e2005-04-16 15:20:36 -07002878/*-----------------------------------------------------------------
2879 * Event notification.
2880 *---------------------------------------------------------------*/
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00002881int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
Milan Broz60935eb2009-06-22 10:12:30 +01002882 unsigned cookie)
Alasdair G Kergon69267a32007-12-13 14:15:57 +00002883{
Milan Broz60935eb2009-06-22 10:12:30 +01002884 char udev_cookie[DM_COOKIE_LENGTH];
2885 char *envp[] = { udev_cookie, NULL };
2886
2887 if (!cookie)
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00002888 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
Milan Broz60935eb2009-06-22 10:12:30 +01002889 else {
2890 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2891 DM_COOKIE_ENV_VAR_NAME, cookie);
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00002892 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2893 action, envp);
Milan Broz60935eb2009-06-22 10:12:30 +01002894 }
Alasdair G Kergon69267a32007-12-13 14:15:57 +00002895}
2896
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002897uint32_t dm_next_uevent_seq(struct mapped_device *md)
2898{
2899 return atomic_add_return(1, &md->uevent_seq);
2900}
2901
Linus Torvalds1da177e2005-04-16 15:20:36 -07002902uint32_t dm_get_event_nr(struct mapped_device *md)
2903{
2904 return atomic_read(&md->event_nr);
2905}
2906
2907int dm_wait_event(struct mapped_device *md, int event_nr)
2908{
2909 return wait_event_interruptible(md->eventq,
2910 (event_nr != atomic_read(&md->event_nr)));
2911}
2912
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002913void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2914{
2915 unsigned long flags;
2916
2917 spin_lock_irqsave(&md->uevent_lock, flags);
2918 list_add(elist, &md->uevent_list);
2919 spin_unlock_irqrestore(&md->uevent_lock, flags);
2920}
2921
Linus Torvalds1da177e2005-04-16 15:20:36 -07002922/*
2923 * The gendisk is only valid as long as you have a reference
2924 * count on 'md'.
2925 */
2926struct gendisk *dm_disk(struct mapped_device *md)
2927{
2928 return md->disk;
2929}
Sami Tolvanen65ff5b72015-03-18 15:52:14 +00002930EXPORT_SYMBOL_GPL(dm_disk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002931
Milan Broz784aae72009-01-06 03:05:12 +00002932struct kobject *dm_kobject(struct mapped_device *md)
2933{
Mikulas Patocka2995fa72014-01-13 19:37:54 -05002934 return &md->kobj_holder.kobj;
Milan Broz784aae72009-01-06 03:05:12 +00002935}
2936
Milan Broz784aae72009-01-06 03:05:12 +00002937struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2938{
2939 struct mapped_device *md;
2940
Mikulas Patocka2995fa72014-01-13 19:37:54 -05002941 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
Milan Broz784aae72009-01-06 03:05:12 +00002942
Hou Taob9a41d22017-11-01 15:42:36 +08002943 spin_lock(&_minor_lock);
2944 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2945 md = NULL;
2946 goto out;
2947 }
Milan Broz784aae72009-01-06 03:05:12 +00002948 dm_get(md);
Hou Taob9a41d22017-11-01 15:42:36 +08002949out:
2950 spin_unlock(&_minor_lock);
2951
Milan Broz784aae72009-01-06 03:05:12 +00002952 return md;
2953}
2954
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +00002955int dm_suspended_md(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002956{
2957 return test_bit(DMF_SUSPENDED, &md->flags);
2958}
2959
Mike Snitzerffcc3932014-10-28 18:34:52 -04002960int dm_suspended_internally_md(struct mapped_device *md)
2961{
2962 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2963}
2964
Mikulas Patocka2c140a22013-11-01 18:27:41 -04002965int dm_test_deferred_remove_flag(struct mapped_device *md)
2966{
2967 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2968}
2969
Kiyoshi Ueda64dbce52009-12-10 23:52:27 +00002970int dm_suspended(struct dm_target *ti)
2971{
Kiyoshi Uedaecdb2e22010-03-06 02:29:52 +00002972 return dm_suspended_md(dm_table_get_md(ti->table));
Kiyoshi Ueda64dbce52009-12-10 23:52:27 +00002973}
2974EXPORT_SYMBOL_GPL(dm_suspended);
2975
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002976int dm_noflush_suspending(struct dm_target *ti)
2977{
Kiyoshi Uedaecdb2e22010-03-06 02:29:52 +00002978 return __noflush_suspending(dm_table_get_md(ti->table));
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002979}
2980EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2981
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002982struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
Mike Snitzer0776aa02017-12-08 14:40:52 -05002983 unsigned integrity, unsigned per_io_data_size,
2984 unsigned min_pool_size)
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002985{
Mike Snitzer115485e2016-02-22 12:16:21 -05002986 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
Mike Snitzer78d8e582015-06-26 10:01:13 -04002987 unsigned int pool_size = 0;
Mike Snitzer64f52b02017-12-11 23:17:47 -05002988 unsigned int front_pad, io_front_pad;
Kent Overstreet6f1c8192018-05-20 18:25:53 -04002989 int ret;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002990
2991 if (!pools)
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04002992 return NULL;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002993
Mike Snitzer78d8e582015-06-26 10:01:13 -04002994 switch (type) {
2995 case DM_TYPE_BIO_BASED:
Toshi Kani545ed202016-06-22 17:54:53 -06002996 case DM_TYPE_DAX_BIO_BASED:
Mike Snitzer22c11852017-12-04 21:07:37 -05002997 case DM_TYPE_NVME_BIO_BASED:
Mike Snitzer0776aa02017-12-08 14:40:52 -05002998 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
Mike Snitzer30187e12016-01-31 13:28:26 -05002999 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
Mike Snitzer64f52b02017-12-11 23:17:47 -05003000 io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
Kent Overstreet6f1c8192018-05-20 18:25:53 -04003001 ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
3002 if (ret)
Mike Snitzer64f52b02017-12-11 23:17:47 -05003003 goto out;
Kent Overstreet6f1c8192018-05-20 18:25:53 -04003004 if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
Christoph Hellwigeb8db832017-01-22 18:32:46 +01003005 goto out;
Mike Snitzer78d8e582015-06-26 10:01:13 -04003006 break;
3007 case DM_TYPE_REQUEST_BASED:
Mike Snitzer0776aa02017-12-08 14:40:52 -05003008 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
Mike Snitzer78d8e582015-06-26 10:01:13 -04003009 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
Mike Snitzer591ddcf2016-01-31 12:05:42 -05003010 /* per_io_data_size is used for blk-mq pdu at queue allocation */
Mike Snitzer78d8e582015-06-26 10:01:13 -04003011 break;
3012 default:
3013 BUG();
3014 }
3015
Kent Overstreet6f1c8192018-05-20 18:25:53 -04003016 ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
3017 if (ret)
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003018 goto out;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003019
Kent Overstreet6f1c8192018-05-20 18:25:53 -04003020 if (integrity && bioset_integrity_create(&pools->bs, pool_size))
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003021 goto out;
Martin K. Petersena91a2782011-03-17 11:11:05 +01003022
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003023 return pools;
Mike Snitzer78d8e582015-06-26 10:01:13 -04003024
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00003025out:
3026 dm_free_md_mempools(pools);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003027
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04003028 return NULL;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003029}
3030
3031void dm_free_md_mempools(struct dm_md_mempools *pools)
3032{
3033 if (!pools)
3034 return;
3035
Kent Overstreet6f1c8192018-05-20 18:25:53 -04003036 bioset_exit(&pools->bs);
3037 bioset_exit(&pools->io_bs);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01003038
3039 kfree(pools);
3040}
3041
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09003042struct dm_pr {
3043 u64 old_key;
3044 u64 new_key;
3045 u32 flags;
3046 bool fail_early;
3047};
3048
3049static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
3050 void *data)
3051{
3052 struct mapped_device *md = bdev->bd_disk->private_data;
3053 struct dm_table *table;
3054 struct dm_target *ti;
3055 int ret = -ENOTTY, srcu_idx;
3056
3057 table = dm_get_live_table(md, &srcu_idx);
3058 if (!table || !dm_table_get_size(table))
3059 goto out;
3060
3061 /* We only support devices that have a single target */
3062 if (dm_table_get_num_targets(table) != 1)
3063 goto out;
3064 ti = dm_table_get_target(table, 0);
3065
3066 ret = -EINVAL;
3067 if (!ti->type->iterate_devices)
3068 goto out;
3069
3070 ret = ti->type->iterate_devices(ti, fn, data);
3071out:
3072 dm_put_live_table(md, srcu_idx);
3073 return ret;
3074}
3075
3076/*
3077 * For register / unregister we need to manually call out to every path.
3078 */
3079static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
3080 sector_t start, sector_t len, void *data)
3081{
3082 struct dm_pr *pr = data;
3083 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3084
3085 if (!ops || !ops->pr_register)
3086 return -EOPNOTSUPP;
3087 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
3088}
3089
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003090static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
Mike Snitzer956a4022016-02-18 16:13:51 -05003091 u32 flags)
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003092{
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09003093 struct dm_pr pr = {
3094 .old_key = old_key,
3095 .new_key = new_key,
3096 .flags = flags,
3097 .fail_early = true,
3098 };
3099 int ret;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003100
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09003101 ret = dm_call_pr(bdev, __dm_pr_register, &pr);
3102 if (ret && new_key) {
3103 /* unregister all paths if we failed to register any path */
3104 pr.old_key = new_key;
3105 pr.new_key = 0;
3106 pr.flags = 0;
3107 pr.fail_early = false;
3108 dm_call_pr(bdev, __dm_pr_register, &pr);
3109 }
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003110
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09003111 return ret;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003112}
3113
3114static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
Mike Snitzer956a4022016-02-18 16:13:51 -05003115 u32 flags)
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003116{
3117 struct mapped_device *md = bdev->bd_disk->private_data;
3118 const struct pr_ops *ops;
Mike Snitzer971888c2018-04-03 15:05:12 -04003119 int r, srcu_idx;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003120
Mike Snitzer5bd5e8d2018-04-03 16:54:10 -04003121 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003122 if (r < 0)
Mike Snitzer971888c2018-04-03 15:05:12 -04003123 goto out;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003124
3125 ops = bdev->bd_disk->fops->pr_ops;
3126 if (ops && ops->pr_reserve)
3127 r = ops->pr_reserve(bdev, key, type, flags);
3128 else
3129 r = -EOPNOTSUPP;
Mike Snitzer971888c2018-04-03 15:05:12 -04003130out:
3131 dm_unprepare_ioctl(md, srcu_idx);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003132 return r;
3133}
3134
3135static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3136{
3137 struct mapped_device *md = bdev->bd_disk->private_data;
3138 const struct pr_ops *ops;
Mike Snitzer971888c2018-04-03 15:05:12 -04003139 int r, srcu_idx;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003140
Mike Snitzer5bd5e8d2018-04-03 16:54:10 -04003141 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003142 if (r < 0)
Mike Snitzer971888c2018-04-03 15:05:12 -04003143 goto out;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003144
3145 ops = bdev->bd_disk->fops->pr_ops;
3146 if (ops && ops->pr_release)
3147 r = ops->pr_release(bdev, key, type);
3148 else
3149 r = -EOPNOTSUPP;
Mike Snitzer971888c2018-04-03 15:05:12 -04003150out:
3151 dm_unprepare_ioctl(md, srcu_idx);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003152 return r;
3153}
3154
3155static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
Mike Snitzer956a4022016-02-18 16:13:51 -05003156 enum pr_type type, bool abort)
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003157{
3158 struct mapped_device *md = bdev->bd_disk->private_data;
3159 const struct pr_ops *ops;
Mike Snitzer971888c2018-04-03 15:05:12 -04003160 int r, srcu_idx;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003161
Mike Snitzer5bd5e8d2018-04-03 16:54:10 -04003162 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003163 if (r < 0)
Mike Snitzer971888c2018-04-03 15:05:12 -04003164 goto out;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003165
3166 ops = bdev->bd_disk->fops->pr_ops;
3167 if (ops && ops->pr_preempt)
3168 r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3169 else
3170 r = -EOPNOTSUPP;
Mike Snitzer971888c2018-04-03 15:05:12 -04003171out:
3172 dm_unprepare_ioctl(md, srcu_idx);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003173 return r;
3174}
3175
3176static int dm_pr_clear(struct block_device *bdev, u64 key)
3177{
3178 struct mapped_device *md = bdev->bd_disk->private_data;
3179 const struct pr_ops *ops;
Mike Snitzer971888c2018-04-03 15:05:12 -04003180 int r, srcu_idx;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003181
Mike Snitzer5bd5e8d2018-04-03 16:54:10 -04003182 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003183 if (r < 0)
Mike Snitzer971888c2018-04-03 15:05:12 -04003184 goto out;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003185
3186 ops = bdev->bd_disk->fops->pr_ops;
3187 if (ops && ops->pr_clear)
3188 r = ops->pr_clear(bdev, key);
3189 else
3190 r = -EOPNOTSUPP;
Mike Snitzer971888c2018-04-03 15:05:12 -04003191out:
3192 dm_unprepare_ioctl(md, srcu_idx);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003193 return r;
3194}
3195
3196static const struct pr_ops dm_pr_ops = {
3197 .pr_register = dm_pr_register,
3198 .pr_reserve = dm_pr_reserve,
3199 .pr_release = dm_pr_release,
3200 .pr_preempt = dm_pr_preempt,
3201 .pr_clear = dm_pr_clear,
3202};
3203
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07003204static const struct block_device_operations dm_blk_dops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003205 .open = dm_blk_open,
3206 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07003207 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08003208 .getgeo = dm_blk_getgeo,
Christoph Hellwige76239a2018-10-12 19:08:49 +09003209 .report_zones = dm_blk_report_zones,
Christoph Hellwig71cdb692015-10-15 14:10:51 +02003210 .pr_ops = &dm_pr_ops,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003211 .owner = THIS_MODULE
3212};
3213
Dan Williamsf26c5712017-04-12 12:35:44 -07003214static const struct dax_operations dm_dax_ops = {
3215 .direct_access = dm_dax_direct_access,
Dan Williams7bf7eac2019-05-16 13:26:29 -07003216 .dax_supported = dm_dax_supported,
Dan Williams7e026c82017-05-29 12:57:56 -07003217 .copy_from_iter = dm_dax_copy_from_iter,
Dan Williamsb3a9a0c2018-05-02 06:46:33 -07003218 .copy_to_iter = dm_dax_copy_to_iter,
Dan Williamsf26c5712017-04-12 12:35:44 -07003219};
3220
Linus Torvalds1da177e2005-04-16 15:20:36 -07003221/*
3222 * module hooks
3223 */
3224module_init(dm_init);
3225module_exit(dm_exit);
3226
3227module_param(major, uint, 0);
3228MODULE_PARM_DESC(major, "The major number of the device mapper");
Mike Snitzerf4790822013-09-12 18:06:12 -04003229
Mike Snitzere8603132013-09-12 18:06:12 -04003230module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3231MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3232
Mike Snitzer115485e2016-02-22 12:16:21 -05003233module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
3234MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3235
Linus Torvalds1da177e2005-04-16 15:20:36 -07003236MODULE_DESCRIPTION(DM_NAME " driver");
3237MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3238MODULE_LICENSE("GPL");