blob: 04477697ee4b4df8de9edb403ffa45b8286fe345 [file] [log] [blame]
Christoph Hellwig3dcf60b2019-04-30 14:42:43 -04001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
5 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
6 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
Jens Axboe6728cb02008-01-31 13:03:55 +01007 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
8 * - July2000
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
10 */
11
12/*
13 * This handles all read/write requests to block devices
14 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/backing-dev.h>
18#include <linux/bio.h>
19#include <linux/blkdev.h>
Jens Axboe320ae512013-10-24 09:20:05 +010020#include <linux/blk-mq.h>
Alan Stern52abca62020-12-08 21:29:51 -080021#include <linux/blk-pm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/highmem.h>
23#include <linux/mm.h>
Matthew Wilcox (Oracle)cee9a0c2020-06-01 21:46:07 -070024#include <linux/pagemap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/kernel_stat.h>
26#include <linux/string.h>
27#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/completion.h>
29#include <linux/slab.h>
30#include <linux/swap.h>
31#include <linux/writeback.h>
Andrew Mortonfaccbd4b2006-12-10 02:19:35 -080032#include <linux/task_io_accounting_ops.h>
Akinobu Mitac17bb492006-12-08 02:39:46 -080033#include <linux/fault-inject.h>
Jens Axboe73c10102011-03-08 13:19:51 +010034#include <linux/list_sort.h>
Tejun Heoe3c78ca2011-10-19 14:32:38 +020035#include <linux/delay.h>
Tejun Heoaaf7c682012-04-19 16:29:22 -070036#include <linux/ratelimit.h>
Lin Ming6c954662013-03-23 11:42:26 +080037#include <linux/pm_runtime.h>
Tejun Heoeea8f412015-05-22 17:13:17 -040038#include <linux/blk-cgroup.h>
Max Gurtovoy54d4e6a2019-09-16 18:44:29 +030039#include <linux/t10-pi.h>
Omar Sandoval18fbda92017-01-31 14:53:20 -080040#include <linux/debugfs.h>
Howard McLauchlan30abb3a2018-02-06 14:05:39 -080041#include <linux/bpf.h>
Johannes Weinerb8e24a92019-08-08 15:03:00 -040042#include <linux/psi.h>
Ming Lei71ac8602020-05-14 16:45:09 +080043#include <linux/sched/sysctl.h>
Satya Tangiralaa892c8d2020-05-14 00:37:18 +000044#include <linux/blk-crypto.h>
Li Zefan55782132009-06-09 13:43:05 +080045
46#define CREATE_TRACE_POINTS
47#include <trace/events/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
Jens Axboe8324aa92008-01-29 14:51:59 +010049#include "blk.h"
Ming Lei43a5e4e2013-12-26 21:31:35 +080050#include "blk-mq.h"
Jens Axboebd166ef2017-01-17 06:03:22 -070051#include "blk-mq-sched.h"
Bart Van Asschebca6b062018-09-26 14:01:03 -070052#include "blk-pm.h"
Josef Bacikc1c80382018-07-03 11:14:59 -040053#include "blk-rq-qos.h"
Jens Axboe8324aa92008-01-29 14:51:59 +010054
Omar Sandoval18fbda92017-01-31 14:53:20 -080055struct dentry *blk_debugfs_root;
Omar Sandoval18fbda92017-01-31 14:53:20 -080056
Mike Snitzerd07335e2010-11-16 12:52:38 +010057EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
Jun'ichi Nomurab0da3f02009-10-01 21:16:13 +020058EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
Linus Torvalds0a82a8d2013-04-18 09:00:26 -070059EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
Keith Busch3291fa52014-04-28 12:30:52 -060060EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
NeilBrowncbae8d42012-12-14 20:49:27 +010061EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
Chaitanya Kulkarnib357e4a2021-02-21 21:29:59 -080062EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert);
Ingo Molnar0bfc2452008-11-26 11:59:56 +010063
Tejun Heoa73f7302011-12-14 00:33:37 +010064DEFINE_IDA(blk_queue_ida);
65
Linus Torvalds1da177e2005-04-16 15:20:36 -070066/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 * For queue allocation
68 */
Jens Axboe6728cb02008-01-31 13:03:55 +010069struct kmem_cache *blk_requestq_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070
71/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * Controlling structure to kblockd
73 */
Jens Axboeff856ba2006-01-09 16:02:34 +010074static struct workqueue_struct *kblockd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070075
Bart Van Assche8814ce82018-03-07 17:10:04 -080076/**
77 * blk_queue_flag_set - atomically set a queue flag
78 * @flag: flag to be set
79 * @q: request queue
80 */
81void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
82{
Christoph Hellwig57d74df2018-11-14 17:02:07 +010083 set_bit(flag, &q->queue_flags);
Bart Van Assche8814ce82018-03-07 17:10:04 -080084}
85EXPORT_SYMBOL(blk_queue_flag_set);
86
87/**
88 * blk_queue_flag_clear - atomically clear a queue flag
89 * @flag: flag to be cleared
90 * @q: request queue
91 */
92void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
93{
Christoph Hellwig57d74df2018-11-14 17:02:07 +010094 clear_bit(flag, &q->queue_flags);
Bart Van Assche8814ce82018-03-07 17:10:04 -080095}
96EXPORT_SYMBOL(blk_queue_flag_clear);
97
98/**
99 * blk_queue_flag_test_and_set - atomically test and set a queue flag
100 * @flag: flag to be set
101 * @q: request queue
102 *
103 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
104 * the flag was already set.
105 */
106bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
107{
Christoph Hellwig57d74df2018-11-14 17:02:07 +0100108 return test_and_set_bit(flag, &q->queue_flags);
Bart Van Assche8814ce82018-03-07 17:10:04 -0800109}
110EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
111
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200112void blk_rq_init(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113{
FUJITA Tomonori1afb20f2008-04-25 12:26:28 +0200114 memset(rq, 0, sizeof(*rq));
115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 INIT_LIST_HEAD(&rq->queuelist);
Jens Axboe63a71382008-02-08 12:41:03 +0100117 rq->q = q;
Tejun Heoa2dec7b2009-05-07 22:24:44 +0900118 rq->__sector = (sector_t) -1;
Jens Axboe2e662b62006-07-13 11:55:04 +0200119 INIT_HLIST_NODE(&rq->hash);
120 RB_CLEAR_NODE(&rq->rb_node);
Xianting Tiane44a6a22020-08-27 14:34:17 +0800121 rq->tag = BLK_MQ_NO_TAG;
122 rq->internal_tag = BLK_MQ_NO_TAG;
Omar Sandoval522a7772018-05-09 02:08:53 -0700123 rq->start_time_ns = ktime_get_ns();
Jerome Marchand09e099d2011-01-05 16:57:38 +0100124 rq->part = NULL;
Josef Bacikb554db12019-03-07 21:37:18 +0000125 refcount_set(&rq->ref, 1);
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000126 blk_crypto_rq_set_defaults(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127}
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200128EXPORT_SYMBOL(blk_rq_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
Chaitanya Kulkarnie47bc4e2019-06-20 10:59:16 -0700130#define REQ_OP_NAME(name) [REQ_OP_##name] = #name
131static const char *const blk_op_name[] = {
132 REQ_OP_NAME(READ),
133 REQ_OP_NAME(WRITE),
134 REQ_OP_NAME(FLUSH),
135 REQ_OP_NAME(DISCARD),
136 REQ_OP_NAME(SECURE_ERASE),
137 REQ_OP_NAME(ZONE_RESET),
Chaitanya Kulkarni6e33dbf2019-08-01 10:26:36 -0700138 REQ_OP_NAME(ZONE_RESET_ALL),
Ajay Joshi6c1b1da2019-10-27 23:05:45 +0900139 REQ_OP_NAME(ZONE_OPEN),
140 REQ_OP_NAME(ZONE_CLOSE),
141 REQ_OP_NAME(ZONE_FINISH),
Keith Busch0512a752020-05-12 17:55:47 +0900142 REQ_OP_NAME(ZONE_APPEND),
Chaitanya Kulkarnie47bc4e2019-06-20 10:59:16 -0700143 REQ_OP_NAME(WRITE_SAME),
144 REQ_OP_NAME(WRITE_ZEROES),
Chaitanya Kulkarnie47bc4e2019-06-20 10:59:16 -0700145 REQ_OP_NAME(DRV_IN),
146 REQ_OP_NAME(DRV_OUT),
147};
148#undef REQ_OP_NAME
149
150/**
151 * blk_op_str - Return string XXX in the REQ_OP_XXX.
152 * @op: REQ_OP_XXX.
153 *
154 * Description: Centralize block layer function to convert REQ_OP_XXX into
155 * string format. Useful in the debugging and tracing bio or request. For
156 * invalid REQ_OP_XXX it returns string "UNKNOWN".
157 */
158inline const char *blk_op_str(unsigned int op)
159{
160 const char *op_str = "UNKNOWN";
161
162 if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
163 op_str = blk_op_name[op];
164
165 return op_str;
166}
167EXPORT_SYMBOL_GPL(blk_op_str);
168
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200169static const struct {
170 int errno;
171 const char *name;
172} blk_errors[] = {
173 [BLK_STS_OK] = { 0, "" },
174 [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" },
175 [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" },
176 [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" },
177 [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" },
178 [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" },
179 [BLK_STS_NEXUS] = { -EBADE, "critical nexus" },
180 [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" },
181 [BLK_STS_PROTECTION] = { -EILSEQ, "protection" },
182 [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" },
Ming Lei86ff7c22018-01-30 22:04:57 -0500183 [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" },
Goldwyn Rodrigues03a07c92017-06-20 07:05:46 -0500184 [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" },
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200185
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200186 /* device mapper special case, should not leak out: */
187 [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" },
188
Keith Busch3b481d92020-09-24 13:53:28 -0700189 /* zone device specific errors */
190 [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" },
191 [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" },
192
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200193 /* everything else not covered above: */
194 [BLK_STS_IOERR] = { -EIO, "I/O" },
195};
196
197blk_status_t errno_to_blk_status(int errno)
198{
199 int i;
200
201 for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
202 if (blk_errors[i].errno == errno)
203 return (__force blk_status_t)i;
204 }
205
206 return BLK_STS_IOERR;
207}
208EXPORT_SYMBOL_GPL(errno_to_blk_status);
209
210int blk_status_to_errno(blk_status_t status)
211{
212 int idx = (__force int)status;
213
Bart Van Assche34bd9c12017-06-21 10:55:46 -0700214 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200215 return -EIO;
216 return blk_errors[idx].errno;
217}
218EXPORT_SYMBOL_GPL(blk_status_to_errno);
219
Christoph Hellwig178cc592019-06-20 10:59:15 -0700220static void print_req_error(struct request *req, blk_status_t status,
221 const char *caller)
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200222{
223 int idx = (__force int)status;
224
Bart Van Assche34bd9c12017-06-21 10:55:46 -0700225 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200226 return;
227
Christoph Hellwig178cc592019-06-20 10:59:15 -0700228 printk_ratelimited(KERN_ERR
Chaitanya Kulkarnib0e51682019-06-20 10:59:18 -0700229 "%s: %s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
230 "phys_seg %u prio class %u\n",
Christoph Hellwig178cc592019-06-20 10:59:15 -0700231 caller, blk_errors[idx].name,
Chaitanya Kulkarnib0e51682019-06-20 10:59:18 -0700232 req->rq_disk ? req->rq_disk->disk_name : "?",
233 blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)),
234 req->cmd_flags & ~REQ_OP_MASK,
235 req->nr_phys_segments,
236 IOPRIO_PRIO_CLASS(req->ioprio));
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200237}
238
NeilBrown5bb23a62007-09-27 12:46:13 +0200239static void req_bio_endio(struct request *rq, struct bio *bio,
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200240 unsigned int nbytes, blk_status_t error)
Tejun Heo797e7db2006-01-06 09:51:03 +0100241{
Mike Snitzer78d8e582015-06-26 10:01:13 -0400242 if (error)
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200243 bio->bi_status = error;
Tejun Heo797e7db2006-01-06 09:51:03 +0100244
Christoph Hellwige8064022016-10-20 15:12:13 +0200245 if (unlikely(rq->rq_flags & RQF_QUIET))
Jens Axboeb7c44ed2015-07-24 12:37:59 -0600246 bio_set_flag(bio, BIO_QUIET);
Tejun Heo143a87f2011-01-25 12:43:52 +0100247
Kent Overstreetf79ea412012-09-20 16:38:30 -0700248 bio_advance(bio, nbytes);
Tejun Heo143a87f2011-01-25 12:43:52 +0100249
Keith Busch0512a752020-05-12 17:55:47 +0900250 if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {
251 /*
252 * Partial zone append completions cannot be supported as the
253 * BIO fragments may end up not being written sequentially.
254 */
255 if (bio->bi_iter.bi_size)
256 bio->bi_status = BLK_STS_IOERR;
257 else
258 bio->bi_iter.bi_sector = rq->__sector;
259 }
260
Tejun Heo143a87f2011-01-25 12:43:52 +0100261 /* don't actually finish bio if it's part of flush sequence */
Christoph Hellwige8064022016-10-20 15:12:13 +0200262 if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200263 bio_endio(bio);
Tejun Heo797e7db2006-01-06 09:51:03 +0100264}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266void blk_dump_rq_flags(struct request *rq, char *msg)
267{
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100268 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
269 rq->rq_disk ? rq->rq_disk->disk_name : "?",
Jens Axboe59533162013-05-23 12:25:08 +0200270 (unsigned long long) rq->cmd_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271
Tejun Heo83096eb2009-05-07 22:24:39 +0900272 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
273 (unsigned long long)blk_rq_pos(rq),
274 blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
Jens Axboeb4f42e22014-04-10 09:46:28 -0600275 printk(KERN_INFO " bio %p, biotail %p, len %u\n",
276 rq->bio, rq->biotail, blk_rq_bytes(rq));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278EXPORT_SYMBOL(blk_dump_rq_flags);
279
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280/**
281 * blk_sync_queue - cancel any pending callbacks on a queue
282 * @q: the queue
283 *
284 * Description:
285 * The block layer may perform asynchronous callback activity
286 * on a queue, such as calling the unplug function after a timeout.
287 * A block device may call blk_sync_queue to ensure that any
288 * such activity is cancelled, thus allowing it to release resources
Michael Opdenacker59c51592007-05-09 08:57:56 +0200289 * that the callbacks might use. The caller must already have made sure
Christoph Hellwigc62b37d2020-07-01 10:59:43 +0200290 * that its ->submit_bio will not re-add plugging prior to calling
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 * this function.
292 *
Vivek Goyalda527772011-03-02 19:05:33 -0500293 * This function does not cancel any asynchronous activity arising
Masanari Iidada3dae52014-09-09 01:27:23 +0900294 * out of elevator or throttling code. That would require elevator_exit()
Tejun Heo5efd6112012-03-05 13:15:12 -0800295 * and blkcg_exit_queue() to be called with queue lock initialized.
Vivek Goyalda527772011-03-02 19:05:33 -0500296 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 */
298void blk_sync_queue(struct request_queue *q)
299{
Jens Axboe70ed28b2008-11-19 14:38:39 +0100300 del_timer_sync(&q->timeout);
Bart Van Assche4e9b6f22017-10-19 10:00:48 -0700301 cancel_work_sync(&q->timeout_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302}
303EXPORT_SYMBOL(blk_sync_queue);
304
305/**
Bart Van Asschecd84a622018-09-26 14:01:04 -0700306 * blk_set_pm_only - increment pm_only counter
Bart Van Asschec9254f22017-11-09 10:49:57 -0800307 * @q: request queue pointer
Bart Van Asschec9254f22017-11-09 10:49:57 -0800308 */
Bart Van Asschecd84a622018-09-26 14:01:04 -0700309void blk_set_pm_only(struct request_queue *q)
Bart Van Asschec9254f22017-11-09 10:49:57 -0800310{
Bart Van Asschecd84a622018-09-26 14:01:04 -0700311 atomic_inc(&q->pm_only);
Bart Van Asschec9254f22017-11-09 10:49:57 -0800312}
Bart Van Asschecd84a622018-09-26 14:01:04 -0700313EXPORT_SYMBOL_GPL(blk_set_pm_only);
Bart Van Asschec9254f22017-11-09 10:49:57 -0800314
Bart Van Asschecd84a622018-09-26 14:01:04 -0700315void blk_clear_pm_only(struct request_queue *q)
Bart Van Asschec9254f22017-11-09 10:49:57 -0800316{
Bart Van Asschecd84a622018-09-26 14:01:04 -0700317 int pm_only;
318
319 pm_only = atomic_dec_return(&q->pm_only);
320 WARN_ON_ONCE(pm_only < 0);
321 if (pm_only == 0)
322 wake_up_all(&q->mq_freeze_wq);
Bart Van Asschec9254f22017-11-09 10:49:57 -0800323}
Bart Van Asschecd84a622018-09-26 14:01:04 -0700324EXPORT_SYMBOL_GPL(blk_clear_pm_only);
Bart Van Asschec9254f22017-11-09 10:49:57 -0800325
Luis Chamberlainb5bd3572020-06-19 20:47:23 +0000326/**
327 * blk_put_queue - decrement the request_queue refcount
328 * @q: the request_queue structure to decrement the refcount for
329 *
330 * Decrements the refcount of the request_queue kobject. When this reaches 0
331 * we'll have blk_release_queue() called.
Luis Chamberlaine8c7d142020-06-19 20:47:25 +0000332 *
333 * Context: Any context, but the last reference must not be dropped from
334 * atomic context.
Luis Chamberlainb5bd3572020-06-19 20:47:23 +0000335 */
Jens Axboe165125e2007-07-24 09:28:11 +0200336void blk_put_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500337{
338 kobject_put(&q->kobj);
339}
Jens Axboed86e0e82011-05-27 07:44:43 +0200340EXPORT_SYMBOL(blk_put_queue);
Al Viro483f4af2006-03-18 18:34:37 -0500341
Jens Axboeaed3ea92014-12-22 14:04:42 -0700342void blk_set_queue_dying(struct request_queue *q)
343{
Bart Van Assche8814ce82018-03-07 17:10:04 -0800344 blk_queue_flag_set(QUEUE_FLAG_DYING, q);
Jens Axboeaed3ea92014-12-22 14:04:42 -0700345
Ming Leid3cfb2a2017-03-27 20:06:58 +0800346 /*
347 * When queue DYING flag is set, we need to block new req
348 * entering queue, so we call blk_freeze_queue_start() to
349 * prevent I/O from crossing blk_queue_enter().
350 */
351 blk_freeze_queue_start(q);
352
Jens Axboe344e9ff2018-11-15 12:22:51 -0700353 if (queue_is_mq(q))
Jens Axboeaed3ea92014-12-22 14:04:42 -0700354 blk_mq_wake_waiters(q);
Ming Lei055f6e12017-11-09 10:49:53 -0800355
356 /* Make blk_queue_enter() reexamine the DYING flag. */
357 wake_up_all(&q->mq_freeze_wq);
Jens Axboeaed3ea92014-12-22 14:04:42 -0700358}
359EXPORT_SYMBOL_GPL(blk_set_queue_dying);
360
Tejun Heod7325802012-03-05 13:14:58 -0800361/**
Tejun Heoc9a929d2011-10-19 14:42:16 +0200362 * blk_cleanup_queue - shutdown a request queue
363 * @q: request queue to shutdown
364 *
Bart Van Asschec246e802012-12-06 14:32:01 +0100365 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
366 * put it. All future requests will be failed immediately with -ENODEV.
Luis Chamberlaine8c7d142020-06-19 20:47:25 +0000367 *
368 * Context: can sleep
Vivek Goyalc94a96a2011-03-02 19:04:42 -0500369 */
Jens Axboe6728cb02008-01-31 13:03:55 +0100370void blk_cleanup_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500371{
Luis Chamberlaine8c7d142020-06-19 20:47:25 +0000372 /* cannot be called from atomic context */
373 might_sleep();
374
Bart Van Asschebae85c12019-09-30 16:00:43 -0700375 WARN_ON_ONCE(blk_queue_registered(q));
376
Bart Van Assche3f3299d2012-11-28 13:42:38 +0100377 /* mark @q DYING, no new request or merges will be allowed afterwards */
Jens Axboeaed3ea92014-12-22 14:04:42 -0700378 blk_set_queue_dying(q);
Tejun Heo6ecf23a2012-03-05 13:14:59 -0800379
Christoph Hellwig57d74df2018-11-14 17:02:07 +0100380 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
381 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
Tejun Heoc9a929d2011-10-19 14:42:16 +0200382
Bart Van Asschec246e802012-12-06 14:32:01 +0100383 /*
384 * Drain all requests queued before DYING marking. Set DEAD flag to
Bart Van Assche67ed8b72019-08-01 15:39:55 -0700385 * prevent that blk_mq_run_hw_queues() accesses the hardware queues
386 * after draining finished.
Bart Van Asschec246e802012-12-06 14:32:01 +0100387 */
Dan Williams3ef28e82015-10-21 13:20:12 -0400388 blk_freeze_queue(q);
Ming Leic57cdf72018-10-24 21:18:09 +0800389
390 rq_qos_exit(q);
391
Christoph Hellwig57d74df2018-11-14 17:02:07 +0100392 blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
Tejun Heoc9a929d2011-10-19 14:42:16 +0200393
Dan Williams5a48fc12015-10-21 13:20:23 -0400394 /* for synchronous bio-based driver finish in-flight integrity i/o */
395 blk_flush_integrity();
396
Tejun Heoc9a929d2011-10-19 14:42:16 +0200397 /* @q won't process any more request, flush async actions */
Jan Karadc3b17c2017-02-02 15:56:50 +0100398 del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
Tejun Heoc9a929d2011-10-19 14:42:16 +0200399 blk_sync_queue(q);
400
Jens Axboe344e9ff2018-11-15 12:22:51 -0700401 if (queue_is_mq(q))
Ming Leic7e2d942019-04-30 09:52:25 +0800402 blk_mq_exit_queue(q);
Jens Axboea1ce35f2018-10-29 10:23:51 -0600403
Ming Leic3e22192019-06-04 21:08:02 +0800404 /*
405 * In theory, request pool of sched_tags belongs to request queue.
406 * However, the current implementation requires tag_set for freeing
407 * requests, so free the pool now.
408 *
409 * Queue has become frozen, there can't be any in-queue requests, so
410 * it is safe to free requests now.
411 */
412 mutex_lock(&q->sysfs_lock);
413 if (q->elevator)
414 blk_mq_sched_free_requests(q);
415 mutex_unlock(&q->sysfs_lock);
416
Dan Williams3ef28e82015-10-21 13:20:12 -0400417 percpu_ref_exit(&q->q_usage_counter);
Bart Van Assche45a9c9d2014-12-09 16:57:48 +0100418
Tejun Heoc9a929d2011-10-19 14:42:16 +0200419 /* @q is and will stay empty, shutdown and put */
Al Viro483f4af2006-03-18 18:34:37 -0500420 blk_put_queue(q);
421}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422EXPORT_SYMBOL(blk_cleanup_queue);
423
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800424/**
425 * blk_queue_enter() - try to increase q->q_usage_counter
426 * @q: request queue pointer
Bart Van Asschea4d34da2020-12-08 21:29:50 -0800427 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PM
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800428 */
Bart Van Assche9a95e4e2017-11-09 10:49:59 -0800429int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
Dan Williams3ef28e82015-10-21 13:20:12 -0400430{
Bart Van Asschea4d34da2020-12-08 21:29:50 -0800431 const bool pm = flags & BLK_MQ_REQ_PM;
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800432
Dan Williams3ef28e82015-10-21 13:20:12 -0400433 while (true) {
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800434 bool success = false;
Dan Williams3ef28e82015-10-21 13:20:12 -0400435
Bart Van Assche818e0fa2018-03-19 11:46:13 -0700436 rcu_read_lock();
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800437 if (percpu_ref_tryget_live(&q->q_usage_counter)) {
438 /*
Bart Van Asschecd84a622018-09-26 14:01:04 -0700439 * The code that increments the pm_only counter is
440 * responsible for ensuring that that counter is
441 * globally visible before the queue is unfrozen.
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800442 */
Alan Stern52abca62020-12-08 21:29:51 -0800443 if ((pm && queue_rpm_status(q) != RPM_SUSPENDED) ||
444 !blk_queue_pm_only(q)) {
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800445 success = true;
446 } else {
447 percpu_ref_put(&q->q_usage_counter);
448 }
449 }
Bart Van Assche818e0fa2018-03-19 11:46:13 -0700450 rcu_read_unlock();
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800451
452 if (success)
Dan Williams3ef28e82015-10-21 13:20:12 -0400453 return 0;
454
Bart Van Assche3a0a5292017-11-09 10:49:58 -0800455 if (flags & BLK_MQ_REQ_NOWAIT)
Dan Williams3ef28e82015-10-21 13:20:12 -0400456 return -EBUSY;
457
Ming Lei5ed61d32017-03-27 20:06:56 +0800458 /*
Ming Lei1671d522017-03-27 20:06:57 +0800459 * read pair of barrier in blk_freeze_queue_start(),
Ming Lei5ed61d32017-03-27 20:06:56 +0800460 * we need to order reading __PERCPU_REF_DEAD flag of
Ming Leid3cfb2a2017-03-27 20:06:58 +0800461 * .q_usage_counter and reading .mq_freeze_depth or
462 * queue dying flag, otherwise the following wait may
463 * never return if the two reads are reordered.
Ming Lei5ed61d32017-03-27 20:06:56 +0800464 */
465 smp_rmb();
466
Alan Jenkins1dc30392018-04-12 19:11:58 +0100467 wait_event(q->mq_freeze_wq,
Bob Liu7996a8b2019-05-21 11:25:55 +0800468 (!q->mq_freeze_depth &&
Alan Stern52abca62020-12-08 21:29:51 -0800469 blk_pm_resume_queue(pm, q)) ||
Alan Jenkins1dc30392018-04-12 19:11:58 +0100470 blk_queue_dying(q));
Dan Williams3ef28e82015-10-21 13:20:12 -0400471 if (blk_queue_dying(q))
472 return -ENODEV;
Dan Williams3ef28e82015-10-21 13:20:12 -0400473 }
474}
475
Christoph Hellwigaccea322020-04-28 13:27:56 +0200476static inline int bio_queue_enter(struct bio *bio)
477{
Christoph Hellwig309dca302021-01-24 11:02:34 +0100478 struct request_queue *q = bio->bi_bdev->bd_disk->queue;
Christoph Hellwigaccea322020-04-28 13:27:56 +0200479 bool nowait = bio->bi_opf & REQ_NOWAIT;
480 int ret;
481
482 ret = blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0);
483 if (unlikely(ret)) {
484 if (nowait && !blk_queue_dying(q))
485 bio_wouldblock_error(bio);
486 else
487 bio_io_error(bio);
488 }
489
490 return ret;
491}
492
Dan Williams3ef28e82015-10-21 13:20:12 -0400493void blk_queue_exit(struct request_queue *q)
494{
495 percpu_ref_put(&q->q_usage_counter);
496}
497
498static void blk_queue_usage_counter_release(struct percpu_ref *ref)
499{
500 struct request_queue *q =
501 container_of(ref, struct request_queue, q_usage_counter);
502
503 wake_up_all(&q->mq_freeze_wq);
504}
505
Kees Cookbca237a2017-08-28 15:03:41 -0700506static void blk_rq_timed_out_timer(struct timer_list *t)
Christoph Hellwig287922e2015-10-30 20:57:30 +0800507{
Kees Cookbca237a2017-08-28 15:03:41 -0700508 struct request_queue *q = from_timer(q, t, timeout);
Christoph Hellwig287922e2015-10-30 20:57:30 +0800509
510 kblockd_schedule_work(&q->timeout_work);
511}
512
Tetsuo Handa2e3c18d2019-01-30 22:21:45 +0900513static void blk_timeout_work(struct work_struct *work)
514{
515}
516
Christoph Hellwigc62b37d2020-07-01 10:59:43 +0200517struct request_queue *blk_alloc_queue(int node_id)
Christoph Lameter19460892005-06-23 00:08:19 -0700518{
Jens Axboe165125e2007-07-24 09:28:11 +0200519 struct request_queue *q;
Kent Overstreet338aa962018-05-20 18:25:47 -0400520 int ret;
Christoph Lameter19460892005-06-23 00:08:19 -0700521
Jens Axboe8324aa92008-01-29 14:51:59 +0100522 q = kmem_cache_alloc_node(blk_requestq_cachep,
Christoph Hellwig3d745ea2020-03-27 09:30:11 +0100523 GFP_KERNEL | __GFP_ZERO, node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 if (!q)
525 return NULL;
526
Christoph Hellwigcbf62af2018-05-31 19:11:36 +0200527 q->last_merge = NULL;
Christoph Hellwigcbf62af2018-05-31 19:11:36 +0200528
Christoph Hellwig3d745ea2020-03-27 09:30:11 +0100529 q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL);
Tejun Heoa73f7302011-12-14 00:33:37 +0100530 if (q->id < 0)
Ming Lei3d2936f2014-05-27 23:35:14 +0800531 goto fail_q;
Tejun Heoa73f7302011-12-14 00:33:37 +0100532
Ming Leic495a172021-01-11 11:05:53 +0800533 ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
Kent Overstreet338aa962018-05-20 18:25:47 -0400534 if (ret)
Kent Overstreet54efd502015-04-23 22:37:18 -0700535 goto fail_id;
536
Christoph Hellwigaef33c22020-05-04 14:48:00 +0200537 q->backing_dev_info = bdi_alloc(node_id);
Jan Karad03f6cd2017-02-02 15:56:51 +0100538 if (!q->backing_dev_info)
Kent Overstreet54efd502015-04-23 22:37:18 -0700539 goto fail_split;
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700540
Jens Axboea83b5762017-03-21 17:20:01 -0600541 q->stats = blk_alloc_queue_stats();
542 if (!q->stats)
543 goto fail_stats;
544
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 q->node = node_id;
546
John Garrybccf5e22020-08-19 23:20:26 +0800547 atomic_set(&q->nr_active_requests_shared_sbitmap, 0);
548
Kees Cookbca237a2017-08-28 15:03:41 -0700549 timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
550 laptop_mode_timer_fn, 0);
551 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
Tetsuo Handa2e3c18d2019-01-30 22:21:45 +0900552 INIT_WORK(&q->timeout_work, blk_timeout_work);
Tejun Heoa612fdd2011-12-14 00:33:41 +0100553 INIT_LIST_HEAD(&q->icq_list);
Tejun Heo4eef3042012-03-05 13:15:18 -0800554#ifdef CONFIG_BLK_CGROUP
Tejun Heoe8989fa2012-03-05 13:15:20 -0800555 INIT_LIST_HEAD(&q->blkg_list);
Tejun Heo4eef3042012-03-05 13:15:18 -0800556#endif
Al Viro483f4af2006-03-18 18:34:37 -0500557
Jens Axboe8324aa92008-01-29 14:51:59 +0100558 kobject_init(&q->kobj, &blk_queue_ktype);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559
Luis Chamberlain85e0cbb2020-06-19 20:47:30 +0000560 mutex_init(&q->debugfs_mutex);
Al Viro483f4af2006-03-18 18:34:37 -0500561 mutex_init(&q->sysfs_lock);
Ming Leicecf5d82019-08-27 19:01:48 +0800562 mutex_init(&q->sysfs_dir_lock);
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700563 spin_lock_init(&q->queue_lock);
Vivek Goyalc94a96a2011-03-02 19:04:42 -0500564
Jens Axboe320ae512013-10-24 09:20:05 +0100565 init_waitqueue_head(&q->mq_freeze_wq);
Bob Liu7996a8b2019-05-21 11:25:55 +0800566 mutex_init(&q->mq_freeze_lock);
Jens Axboe320ae512013-10-24 09:20:05 +0100567
Dan Williams3ef28e82015-10-21 13:20:12 -0400568 /*
569 * Init percpu_ref in atomic mode so that it's faster to shutdown.
570 * See blk_register_queue() for details.
571 */
572 if (percpu_ref_init(&q->q_usage_counter,
573 blk_queue_usage_counter_release,
574 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
Mikulas Patockafff49962013-10-14 12:11:36 -0400575 goto fail_bdi;
Tejun Heof51b8022012-03-05 13:15:05 -0800576
Dan Williams3ef28e82015-10-21 13:20:12 -0400577 if (blkcg_init_queue(q))
578 goto fail_ref;
579
Christoph Hellwig3d745ea2020-03-27 09:30:11 +0100580 blk_queue_dma_alignment(q, 511);
581 blk_set_default_limits(&q->limits);
Christoph Hellwigc62b37d2020-07-01 10:59:43 +0200582 q->nr_requests = BLKDEV_MAX_RQ;
Christoph Hellwig3d745ea2020-03-27 09:30:11 +0100583
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 return q;
Tejun Heoa73f7302011-12-14 00:33:37 +0100585
Dan Williams3ef28e82015-10-21 13:20:12 -0400586fail_ref:
587 percpu_ref_exit(&q->q_usage_counter);
Mikulas Patockafff49962013-10-14 12:11:36 -0400588fail_bdi:
Jens Axboea83b5762017-03-21 17:20:01 -0600589 blk_free_queue_stats(q->stats);
590fail_stats:
Jan Karad03f6cd2017-02-02 15:56:51 +0100591 bdi_put(q->backing_dev_info);
Kent Overstreet54efd502015-04-23 22:37:18 -0700592fail_split:
Kent Overstreet338aa962018-05-20 18:25:47 -0400593 bioset_exit(&q->bio_split);
Tejun Heoa73f7302011-12-14 00:33:37 +0100594fail_id:
595 ida_simple_remove(&blk_queue_ida, q->id);
596fail_q:
597 kmem_cache_free(blk_requestq_cachep, q);
598 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
Luis Chamberlainb5bd3572020-06-19 20:47:23 +0000601/**
602 * blk_get_queue - increment the request_queue refcount
603 * @q: the request_queue structure to increment the refcount for
604 *
605 * Increment the refcount of the request_queue kobject.
Luis Chamberlain763b5892020-06-19 20:47:24 +0000606 *
607 * Context: Any context.
Luis Chamberlainb5bd3572020-06-19 20:47:23 +0000608 */
Tejun Heo09ac46c2011-12-14 00:33:38 +0100609bool blk_get_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610{
Bart Van Assche3f3299d2012-11-28 13:42:38 +0100611 if (likely(!blk_queue_dying(q))) {
Tejun Heo09ac46c2011-12-14 00:33:38 +0100612 __blk_get_queue(q);
613 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 }
615
Tejun Heo09ac46c2011-12-14 00:33:38 +0100616 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617}
Jens Axboed86e0e82011-05-27 07:44:43 +0200618EXPORT_SYMBOL(blk_get_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619
Bart Van Assche6a156742017-11-09 10:49:54 -0800620/**
Christoph Hellwigff005a02018-05-09 09:54:05 +0200621 * blk_get_request - allocate a request
Bart Van Assche6a156742017-11-09 10:49:54 -0800622 * @q: request queue to allocate a request for
623 * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
624 * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
625 */
Christoph Hellwigff005a02018-05-09 09:54:05 +0200626struct request *blk_get_request(struct request_queue *q, unsigned int op,
627 blk_mq_req_flags_t flags)
Jens Axboe320ae512013-10-24 09:20:05 +0100628{
Bart Van Assched280bab2017-06-20 11:15:40 -0700629 struct request *req;
630
Bart Van Assche6a156742017-11-09 10:49:54 -0800631 WARN_ON_ONCE(op & REQ_NOWAIT);
Bart Van Asschea4d34da2020-12-08 21:29:50 -0800632 WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PM));
Bart Van Assche6a156742017-11-09 10:49:54 -0800633
Jens Axboea1ce35f2018-10-29 10:23:51 -0600634 req = blk_mq_alloc_request(q, op, flags);
635 if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
636 q->mq_ops->initialize_rq_fn(req);
Bart Van Assched280bab2017-06-20 11:15:40 -0700637
638 return req;
Jens Axboe320ae512013-10-24 09:20:05 +0100639}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640EXPORT_SYMBOL(blk_get_request);
641
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642void blk_put_request(struct request *req)
643{
Jens Axboea1ce35f2018-10-29 10:23:51 -0600644 blk_mq_free_request(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646EXPORT_SYMBOL(blk_put_request);
647
Christoph Hellwig52c5e622018-03-14 16:56:53 +0100648static void handle_bad_sector(struct bio *bio, sector_t maxsector)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649{
650 char b[BDEVNAME_SIZE];
651
Tetsuo Handaf4ac7122020-10-08 22:37:23 +0900652 pr_info_ratelimited("attempt to access beyond end of device\n"
653 "%s: rw=%d, want=%llu, limit=%llu\n",
654 bio_devname(bio, b), bio->bi_opf,
655 bio_end_sector(bio), maxsector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656}
657
Akinobu Mitac17bb492006-12-08 02:39:46 -0800658#ifdef CONFIG_FAIL_MAKE_REQUEST
659
660static DECLARE_FAULT_ATTR(fail_make_request);
661
662static int __init setup_fail_make_request(char *str)
663{
664 return setup_fault_attr(&fail_make_request, str);
665}
666__setup("fail_make_request=", setup_fail_make_request);
667
Christoph Hellwig8446fe92020-11-24 09:36:54 +0100668static bool should_fail_request(struct block_device *part, unsigned int bytes)
Akinobu Mitac17bb492006-12-08 02:39:46 -0800669{
Christoph Hellwig8446fe92020-11-24 09:36:54 +0100670 return part->bd_make_it_fail && should_fail(&fail_make_request, bytes);
Akinobu Mitac17bb492006-12-08 02:39:46 -0800671}
672
673static int __init fail_make_request_debugfs(void)
674{
Akinobu Mitadd48c082011-08-03 16:21:01 -0700675 struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
676 NULL, &fail_make_request);
677
Duan Jiong21f9fcd2014-04-11 15:58:56 +0800678 return PTR_ERR_OR_ZERO(dir);
Akinobu Mitac17bb492006-12-08 02:39:46 -0800679}
680
681late_initcall(fail_make_request_debugfs);
682
683#else /* CONFIG_FAIL_MAKE_REQUEST */
684
Christoph Hellwig8446fe92020-11-24 09:36:54 +0100685static inline bool should_fail_request(struct block_device *part,
Akinobu Mitab2c9cd32011-07-26 16:09:03 -0700686 unsigned int bytes)
Akinobu Mitac17bb492006-12-08 02:39:46 -0800687{
Akinobu Mitab2c9cd32011-07-26 16:09:03 -0700688 return false;
Akinobu Mitac17bb492006-12-08 02:39:46 -0800689}
690
691#endif /* CONFIG_FAIL_MAKE_REQUEST */
692
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100693static inline bool bio_check_ro(struct bio *bio)
Ilya Dryomov721c7fc2018-01-11 14:09:11 +0100694{
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100695 if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) {
Ilya Dryomov721c7fc2018-01-11 14:09:11 +0100696 char b[BDEVNAME_SIZE];
697
Mikulas Patocka8b2ded12018-09-05 16:14:36 -0600698 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
699 return false;
700
Linus Torvaldsa32e2362018-08-03 12:22:09 -0700701 WARN_ONCE(1,
Christoph Hellwigc8178672020-07-01 10:59:40 +0200702 "Trying to write to read-only block-device %s (partno %d)\n",
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100703 bio_devname(bio, b), bio->bi_bdev->bd_partno);
Linus Torvaldsa32e2362018-08-03 12:22:09 -0700704 /* Older lvm-tools actually trigger this */
705 return false;
Ilya Dryomov721c7fc2018-01-11 14:09:11 +0100706 }
707
708 return false;
709}
710
Howard McLauchlan30abb3a2018-02-06 14:05:39 -0800711static noinline int should_fail_bio(struct bio *bio)
712{
Christoph Hellwig309dca302021-01-24 11:02:34 +0100713 if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size))
Howard McLauchlan30abb3a2018-02-06 14:05:39 -0800714 return -EIO;
715 return 0;
716}
717ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
718
Jens Axboec07e2b42007-07-18 13:27:58 +0200719/*
Christoph Hellwig52c5e622018-03-14 16:56:53 +0100720 * Check whether this bio extends beyond the end of the device or partition.
721 * This may well happen - the kernel calls bread() without checking the size of
722 * the device, e.g., when mounting a file system.
723 */
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100724static inline int bio_check_eod(struct bio *bio)
Christoph Hellwig52c5e622018-03-14 16:56:53 +0100725{
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100726 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
Christoph Hellwig52c5e622018-03-14 16:56:53 +0100727 unsigned int nr_sectors = bio_sectors(bio);
728
729 if (nr_sectors && maxsector &&
730 (nr_sectors > maxsector ||
731 bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
732 handle_bad_sector(bio, maxsector);
733 return -EIO;
734 }
735 return 0;
736}
737
738/*
Christoph Hellwig74d46992017-08-23 19:10:32 +0200739 * Remap block n of partition p to block n+start(p) of the disk.
740 */
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100741static int blk_partition_remap(struct bio *bio)
Christoph Hellwig74d46992017-08-23 19:10:32 +0200742{
Christoph Hellwig309dca302021-01-24 11:02:34 +0100743 struct block_device *p = bio->bi_bdev;
Christoph Hellwig74d46992017-08-23 19:10:32 +0200744
Christoph Hellwig52c5e622018-03-14 16:56:53 +0100745 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100746 return -EIO;
Damien Le Moal5eac3eb2019-11-11 11:39:25 +0900747 if (bio_sectors(bio)) {
Christoph Hellwig8446fe92020-11-24 09:36:54 +0100748 bio->bi_iter.bi_sector += p->bd_start_sect;
Christoph Hellwig1c02fca2020-12-03 17:21:38 +0100749 trace_block_bio_remap(bio, p->bd_dev,
Christoph Hellwig29ff57c2020-11-24 09:34:24 +0100750 bio->bi_iter.bi_sector -
Christoph Hellwig8446fe92020-11-24 09:36:54 +0100751 p->bd_start_sect);
Christoph Hellwig52c5e622018-03-14 16:56:53 +0100752 }
Christoph Hellwig30c5d342021-01-24 11:02:36 +0100753 bio_set_flag(bio, BIO_REMAPPED);
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100754 return 0;
Christoph Hellwig74d46992017-08-23 19:10:32 +0200755}
756
Keith Busch0512a752020-05-12 17:55:47 +0900757/*
758 * Check write append to a zoned block device.
759 */
760static inline blk_status_t blk_check_zone_append(struct request_queue *q,
761 struct bio *bio)
762{
763 sector_t pos = bio->bi_iter.bi_sector;
764 int nr_sectors = bio_sectors(bio);
765
766 /* Only applicable to zoned block devices */
767 if (!blk_queue_is_zoned(q))
768 return BLK_STS_NOTSUPP;
769
770 /* The bio sector must point to the start of a sequential zone */
771 if (pos & (blk_queue_zone_sectors(q) - 1) ||
772 !blk_queue_zone_is_seq(q, pos))
773 return BLK_STS_IOERR;
774
775 /*
776 * Not allowed to cross zone boundaries. Otherwise, the BIO will be
777 * split and could result in non-contiguous sectors being written in
778 * different zones.
779 */
780 if (nr_sectors > q->limits.chunk_sectors)
781 return BLK_STS_IOERR;
782
783 /* Make sure the BIO is small enough and will not get split */
784 if (nr_sectors > q->limits.max_zone_append_sectors)
785 return BLK_STS_IOERR;
786
787 bio->bi_opf |= REQ_NOMERGE;
788
789 return BLK_STS_OK;
790}
791
Christoph Hellwiged00aab2020-07-01 10:59:44 +0200792static noinline_for_stack bool submit_bio_checks(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793{
Christoph Hellwig309dca302021-01-24 11:02:34 +0100794 struct block_device *bdev = bio->bi_bdev;
795 struct request_queue *q = bdev->bd_disk->queue;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200796 blk_status_t status = BLK_STS_IOERR;
Jens Axboe5a473e82020-06-04 11:23:39 -0600797 struct blk_plug *plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798
799 might_sleep();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800
Jens Axboe5a473e82020-06-04 11:23:39 -0600801 plug = blk_mq_plug(q, bio);
802 if (plug && plug->nowait)
803 bio->bi_opf |= REQ_NOWAIT;
804
Goldwyn Rodrigues03a07c92017-06-20 07:05:46 -0500805 /*
Jens Axboeb0beb282020-05-28 13:19:29 -0600806 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
Mike Snitzer021a2442020-09-23 16:06:51 -0400807 * if queue does not support NOWAIT.
Goldwyn Rodrigues03a07c92017-06-20 07:05:46 -0500808 */
Mike Snitzer021a2442020-09-23 16:06:51 -0400809 if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q))
Jens Axboeb0beb282020-05-28 13:19:29 -0600810 goto not_supported;
Goldwyn Rodrigues03a07c92017-06-20 07:05:46 -0500811
Howard McLauchlan30abb3a2018-02-06 14:05:39 -0800812 if (should_fail_bio(bio))
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +0200813 goto end_io;
Christoph Hellwig2f9f6222021-01-24 11:02:35 +0100814 if (unlikely(bio_check_ro(bio)))
815 goto end_io;
Christoph Hellwig3a905c32021-01-25 19:39:57 +0100816 if (!bio_flagged(bio, BIO_REMAPPED)) {
817 if (unlikely(bio_check_eod(bio)))
818 goto end_io;
819 if (bdev->bd_partno && unlikely(blk_partition_remap(bio)))
820 goto end_io;
821 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +0200823 /*
Christoph Hellwiged00aab2020-07-01 10:59:44 +0200824 * Filter flush bio's early so that bio based drivers without flush
825 * support don't have to worry about them.
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +0200826 */
Jens Axboef3a8ab72017-01-27 09:08:23 -0700827 if (op_is_flush(bio->bi_opf) &&
Jens Axboec888a8f2016-04-13 13:33:19 -0600828 !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
Jens Axboe1eff9d32016-08-05 15:35:16 -0600829 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
Christoph Hellwige439ab72020-07-01 10:59:42 +0200830 if (!bio_sectors(bio)) {
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200831 status = BLK_STS_OK;
Tejun Heoa7384672008-11-28 13:32:03 +0900832 goto end_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 }
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +0200834 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835
Christoph Hellwigd04c4062018-12-14 17:21:22 +0100836 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
837 bio->bi_opf &= ~REQ_HIPRI;
838
Christoph Hellwig288dab82016-06-09 16:00:36 +0200839 switch (bio_op(bio)) {
840 case REQ_OP_DISCARD:
841 if (!blk_queue_discard(q))
842 goto not_supported;
843 break;
844 case REQ_OP_SECURE_ERASE:
845 if (!blk_queue_secure_erase(q))
846 goto not_supported;
847 break;
848 case REQ_OP_WRITE_SAME:
Christoph Hellwig74d46992017-08-23 19:10:32 +0200849 if (!q->limits.max_write_same_sectors)
Christoph Hellwig288dab82016-06-09 16:00:36 +0200850 goto not_supported;
Nicolai Stange58886782016-12-04 14:56:39 +0100851 break;
Keith Busch0512a752020-05-12 17:55:47 +0900852 case REQ_OP_ZONE_APPEND:
853 status = blk_check_zone_append(q, bio);
854 if (status != BLK_STS_OK)
855 goto end_io;
856 break;
Shaun Tancheff2d253442016-10-18 15:40:32 +0900857 case REQ_OP_ZONE_RESET:
Ajay Joshi6c1b1da2019-10-27 23:05:45 +0900858 case REQ_OP_ZONE_OPEN:
859 case REQ_OP_ZONE_CLOSE:
860 case REQ_OP_ZONE_FINISH:
Christoph Hellwig74d46992017-08-23 19:10:32 +0200861 if (!blk_queue_is_zoned(q))
Shaun Tancheff2d253442016-10-18 15:40:32 +0900862 goto not_supported;
Christoph Hellwig288dab82016-06-09 16:00:36 +0200863 break;
Chaitanya Kulkarni6e33dbf2019-08-01 10:26:36 -0700864 case REQ_OP_ZONE_RESET_ALL:
865 if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q))
866 goto not_supported;
867 break;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800868 case REQ_OP_WRITE_ZEROES:
Christoph Hellwig74d46992017-08-23 19:10:32 +0200869 if (!q->limits.max_write_zeroes_sectors)
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800870 goto not_supported;
871 break;
Christoph Hellwig288dab82016-06-09 16:00:36 +0200872 default:
873 break;
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +0200874 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875
Tejun Heo7f4b35d2012-06-04 20:40:56 -0700876 /*
Christoph Hellwig3e82c342020-04-25 09:55:51 +0200877 * Various block parts want %current->io_context, so allocate it up
878 * front rather than dealing with lots of pain to allocate it only
879 * where needed. This may fail and the block layer knows how to live
880 * with it.
Tejun Heo7f4b35d2012-06-04 20:40:56 -0700881 */
Christoph Hellwig3e82c342020-04-25 09:55:51 +0200882 if (unlikely(!current->io_context))
883 create_task_io_context(current, GFP_ATOMIC, q->node);
Tejun Heo7f4b35d2012-06-04 20:40:56 -0700884
Christoph Hellwigdb18a532020-06-27 09:31:58 +0200885 if (blk_throtl_bio(bio)) {
886 blkcg_bio_issue_init(bio);
Tejun Heoae118892015-08-18 14:55:20 -0700887 return false;
Christoph Hellwigdb18a532020-06-27 09:31:58 +0200888 }
889
890 blk_cgroup_bio_start(bio);
891 blkcg_bio_issue_init(bio);
Christoph Hellwig27a84d52011-09-15 14:01:40 +0200892
NeilBrownfbbaf702017-04-07 09:40:52 -0600893 if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
Christoph Hellwige8a676d2020-12-03 17:21:36 +0100894 trace_block_bio_queue(bio);
NeilBrownfbbaf702017-04-07 09:40:52 -0600895 /* Now that enqueuing has been traced, we need to trace
896 * completion as well.
897 */
898 bio_set_flag(bio, BIO_TRACE_COMPLETION);
899 }
Christoph Hellwig27a84d52011-09-15 14:01:40 +0200900 return true;
Tejun Heoa7384672008-11-28 13:32:03 +0900901
Christoph Hellwig288dab82016-06-09 16:00:36 +0200902not_supported:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200903 status = BLK_STS_NOTSUPP;
Tejun Heoa7384672008-11-28 13:32:03 +0900904end_io:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200905 bio->bi_status = status;
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200906 bio_endio(bio);
Christoph Hellwig27a84d52011-09-15 14:01:40 +0200907 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908}
909
Christoph Hellwiged00aab2020-07-01 10:59:44 +0200910static blk_qc_t __submit_bio(struct bio *bio)
Christoph Hellwigac7c5672020-05-16 20:28:01 +0200911{
Christoph Hellwig309dca302021-01-24 11:02:34 +0100912 struct gendisk *disk = bio->bi_bdev->bd_disk;
Christoph Hellwigac7c5672020-05-16 20:28:01 +0200913 blk_qc_t ret = BLK_QC_T_NONE;
914
915 if (blk_crypto_bio_prep(&bio)) {
Christoph Hellwigc62b37d2020-07-01 10:59:43 +0200916 if (!disk->fops->submit_bio)
917 return blk_mq_submit_bio(bio);
918 ret = disk->fops->submit_bio(bio);
Christoph Hellwigac7c5672020-05-16 20:28:01 +0200919 }
Christoph Hellwigc62b37d2020-07-01 10:59:43 +0200920 blk_queue_exit(disk->queue);
Christoph Hellwigac7c5672020-05-16 20:28:01 +0200921 return ret;
922}
923
Christoph Hellwig566acf22020-07-01 10:59:45 +0200924/*
925 * The loop in this function may be a bit non-obvious, and so deserves some
926 * explanation:
927 *
928 * - Before entering the loop, bio->bi_next is NULL (as all callers ensure
929 * that), so we have a list with a single bio.
930 * - We pretend that we have just taken it off a longer list, so we assign
931 * bio_list to a pointer to the bio_list_on_stack, thus initialising the
932 * bio_list of new bios to be added. ->submit_bio() may indeed add some more
933 * bios through a recursive call to submit_bio_noacct. If it did, we find a
934 * non-NULL value in bio_list and re-enter the loop from the top.
935 * - In this case we really did just take the bio of the top of the list (no
936 * pretending) and so remove it from bio_list, and call into ->submit_bio()
937 * again.
938 *
939 * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio.
940 * bio_list_on_stack[1] contains bios that were submitted before the current
941 * ->submit_bio_bio, but that haven't been processed yet.
942 */
943static blk_qc_t __submit_bio_noacct(struct bio *bio)
944{
945 struct bio_list bio_list_on_stack[2];
946 blk_qc_t ret = BLK_QC_T_NONE;
947
948 BUG_ON(bio->bi_next);
949
950 bio_list_init(&bio_list_on_stack[0]);
951 current->bio_list = bio_list_on_stack;
952
953 do {
Christoph Hellwig309dca302021-01-24 11:02:34 +0100954 struct request_queue *q = bio->bi_bdev->bd_disk->queue;
Christoph Hellwig566acf22020-07-01 10:59:45 +0200955 struct bio_list lower, same;
956
957 if (unlikely(bio_queue_enter(bio) != 0))
958 continue;
959
960 /*
961 * Create a fresh bio_list for all subordinate requests.
962 */
963 bio_list_on_stack[1] = bio_list_on_stack[0];
964 bio_list_init(&bio_list_on_stack[0]);
965
966 ret = __submit_bio(bio);
967
968 /*
969 * Sort new bios into those for a lower level and those for the
970 * same level.
971 */
972 bio_list_init(&lower);
973 bio_list_init(&same);
974 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
Christoph Hellwig309dca302021-01-24 11:02:34 +0100975 if (q == bio->bi_bdev->bd_disk->queue)
Christoph Hellwig566acf22020-07-01 10:59:45 +0200976 bio_list_add(&same, bio);
977 else
978 bio_list_add(&lower, bio);
979
980 /*
981 * Now assemble so we handle the lowest level first.
982 */
983 bio_list_merge(&bio_list_on_stack[0], &lower);
984 bio_list_merge(&bio_list_on_stack[0], &same);
985 bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
986 } while ((bio = bio_list_pop(&bio_list_on_stack[0])));
987
988 current->bio_list = NULL;
989 return ret;
990}
991
Christoph Hellwigff93ea02020-07-01 10:59:46 +0200992static blk_qc_t __submit_bio_noacct_mq(struct bio *bio)
993{
Christoph Hellwig7c792f32020-07-02 21:21:25 +0200994 struct bio_list bio_list[2] = { };
Christoph Hellwigff93ea02020-07-01 10:59:46 +0200995 blk_qc_t ret = BLK_QC_T_NONE;
996
Christoph Hellwig7c792f32020-07-02 21:21:25 +0200997 current->bio_list = bio_list;
Christoph Hellwigff93ea02020-07-01 10:59:46 +0200998
999 do {
Christoph Hellwig309dca302021-01-24 11:02:34 +01001000 struct gendisk *disk = bio->bi_bdev->bd_disk;
Christoph Hellwigff93ea02020-07-01 10:59:46 +02001001
1002 if (unlikely(bio_queue_enter(bio) != 0))
1003 continue;
1004
1005 if (!blk_crypto_bio_prep(&bio)) {
1006 blk_queue_exit(disk->queue);
1007 ret = BLK_QC_T_NONE;
1008 continue;
1009 }
1010
1011 ret = blk_mq_submit_bio(bio);
Christoph Hellwig7c792f32020-07-02 21:21:25 +02001012 } while ((bio = bio_list_pop(&bio_list[0])));
Christoph Hellwigff93ea02020-07-01 10:59:46 +02001013
1014 current->bio_list = NULL;
1015 return ret;
1016}
1017
Christoph Hellwig27a84d52011-09-15 14:01:40 +02001018/**
Christoph Hellwiged00aab2020-07-01 10:59:44 +02001019 * submit_bio_noacct - re-submit a bio to the block device layer for I/O
Christoph Hellwig27a84d52011-09-15 14:01:40 +02001020 * @bio: The bio describing the location in memory and on the device.
1021 *
Christoph Hellwig3fdd4082020-04-28 13:27:53 +02001022 * This is a version of submit_bio() that shall only be used for I/O that is
1023 * resubmitted to lower level drivers by stacking block drivers. All file
1024 * systems and other upper level users of the block layer should use
1025 * submit_bio() instead.
Neil Brownd89d8792007-05-01 09:53:42 +02001026 */
Christoph Hellwiged00aab2020-07-01 10:59:44 +02001027blk_qc_t submit_bio_noacct(struct bio *bio)
Neil Brownd89d8792007-05-01 09:53:42 +02001028{
Christoph Hellwiged00aab2020-07-01 10:59:44 +02001029 if (!submit_bio_checks(bio))
Christoph Hellwig566acf22020-07-01 10:59:45 +02001030 return BLK_QC_T_NONE;
Christoph Hellwig27a84d52011-09-15 14:01:40 +02001031
1032 /*
Christoph Hellwig566acf22020-07-01 10:59:45 +02001033 * We only want one ->submit_bio to be active at a time, else stack
1034 * usage with stacked devices could be a problem. Use current->bio_list
1035 * to collect a list of requests submited by a ->submit_bio method while
1036 * it is active, and then process them after it returned.
Christoph Hellwig27a84d52011-09-15 14:01:40 +02001037 */
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001038 if (current->bio_list) {
NeilBrownf5fe1b52017-03-10 17:00:47 +11001039 bio_list_add(&current->bio_list[0], bio);
Christoph Hellwig566acf22020-07-01 10:59:45 +02001040 return BLK_QC_T_NONE;
Neil Brownd89d8792007-05-01 09:53:42 +02001041 }
Christoph Hellwig27a84d52011-09-15 14:01:40 +02001042
Christoph Hellwig309dca302021-01-24 11:02:34 +01001043 if (!bio->bi_bdev->bd_disk->fops->submit_bio)
Christoph Hellwigff93ea02020-07-01 10:59:46 +02001044 return __submit_bio_noacct_mq(bio);
Christoph Hellwig566acf22020-07-01 10:59:45 +02001045 return __submit_bio_noacct(bio);
Neil Brownd89d8792007-05-01 09:53:42 +02001046}
Christoph Hellwiged00aab2020-07-01 10:59:44 +02001047EXPORT_SYMBOL(submit_bio_noacct);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048
1049/**
Randy Dunlap710027a2008-08-19 20:13:11 +02001050 * submit_bio - submit a bio to the block device layer for I/O
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051 * @bio: The &struct bio which describes the I/O
1052 *
Christoph Hellwig3fdd4082020-04-28 13:27:53 +02001053 * submit_bio() is used to submit I/O requests to block devices. It is passed a
1054 * fully set up &struct bio that describes the I/O that needs to be done. The
Christoph Hellwig309dca302021-01-24 11:02:34 +01001055 * bio will be send to the device described by the bi_bdev field.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 *
Christoph Hellwig3fdd4082020-04-28 13:27:53 +02001057 * The success/failure status of the request, along with notification of
1058 * completion, is delivered asynchronously through the ->bi_end_io() callback
1059 * in @bio. The bio must NOT be touched by thecaller until ->bi_end_io() has
1060 * been called.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061 */
Mike Christie4e49ea42016-06-05 14:31:41 -05001062blk_qc_t submit_bio(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063{
Tejun Heod3f77df2019-06-27 13:39:52 -07001064 if (blkcg_punt_bio_submit(bio))
1065 return BLK_QC_T_NONE;
1066
Jens Axboebf2de6f2007-09-27 13:01:25 +02001067 /*
1068 * If it's a regular read/write or a barrier with data attached,
1069 * go through the normal accounting stuff before submission.
1070 */
Martin K. Petersene2a60da2012-09-18 12:19:25 -04001071 if (bio_has_data(bio)) {
Martin K. Petersen4363ac72012-09-18 12:19:27 -04001072 unsigned int count;
1073
Mike Christie95fe6c12016-06-05 14:31:48 -05001074 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
Christoph Hellwig309dca302021-01-24 11:02:34 +01001075 count = queue_logical_block_size(
1076 bio->bi_bdev->bd_disk->queue) >> 9;
Martin K. Petersen4363ac72012-09-18 12:19:27 -04001077 else
1078 count = bio_sectors(bio);
1079
Mike Christiea8ebb052016-06-05 14:31:45 -05001080 if (op_is_write(bio_op(bio))) {
Jens Axboebf2de6f2007-09-27 13:01:25 +02001081 count_vm_events(PGPGOUT, count);
1082 } else {
Kent Overstreet4f024f32013-10-11 15:44:27 -07001083 task_io_account_read(bio->bi_iter.bi_size);
Jens Axboebf2de6f2007-09-27 13:01:25 +02001084 count_vm_events(PGPGIN, count);
1085 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086 }
1087
Johannes Weinerb8e24a92019-08-08 15:03:00 -04001088 /*
Christoph Hellwig760f83e2020-04-28 13:27:54 +02001089 * If we're reading data that is part of the userspace workingset, count
1090 * submission time as memory stall. When the device is congested, or
1091 * the submitting cgroup IO-throttled, submission can be a significant
1092 * part of overall IO time.
Johannes Weinerb8e24a92019-08-08 15:03:00 -04001093 */
Christoph Hellwig760f83e2020-04-28 13:27:54 +02001094 if (unlikely(bio_op(bio) == REQ_OP_READ &&
1095 bio_flagged(bio, BIO_WORKINGSET))) {
1096 unsigned long pflags;
1097 blk_qc_t ret;
1098
Johannes Weinerb8e24a92019-08-08 15:03:00 -04001099 psi_memstall_enter(&pflags);
Christoph Hellwiged00aab2020-07-01 10:59:44 +02001100 ret = submit_bio_noacct(bio);
Johannes Weinerb8e24a92019-08-08 15:03:00 -04001101 psi_memstall_leave(&pflags);
1102
Christoph Hellwig760f83e2020-04-28 13:27:54 +02001103 return ret;
1104 }
1105
Christoph Hellwiged00aab2020-07-01 10:59:44 +02001106 return submit_bio_noacct(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108EXPORT_SYMBOL(submit_bio);
1109
Jens Axboe1052b8a2018-11-26 08:21:49 -07001110/**
Hannes Reineckebf4e6b42015-11-26 08:46:57 +01001111 * blk_cloned_rq_check_limits - Helper function to check a cloned request
Guoqing Jiang0d720312020-03-09 22:41:33 +01001112 * for the new queue limits
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001113 * @q: the queue
1114 * @rq: the request being checked
1115 *
1116 * Description:
1117 * @rq may have been made based on weaker limitations of upper-level queues
1118 * in request stacking drivers, and it may violate the limitation of @q.
1119 * Since the block layer and the underlying device driver trust @rq
1120 * after it is inserted to @q, it should be checked against @q before
1121 * the insertion using this generic function.
1122 *
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001123 * Request stacking drivers like request-based dm may change the queue
Hannes Reineckebf4e6b42015-11-26 08:46:57 +01001124 * limits when retrying requests on other queues. Those requests need
1125 * to be checked against the new queue limits again during dispatch.
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001126 */
Ritika Srivastava143d2602020-09-01 13:17:30 -07001127static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q,
Hannes Reineckebf4e6b42015-11-26 08:46:57 +01001128 struct request *rq)
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001129{
Ritika Srivastava8327cce52020-09-01 13:17:31 -07001130 unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
1131
1132 if (blk_rq_sectors(rq) > max_sectors) {
1133 /*
1134 * SCSI device does not have a good way to return if
1135 * Write Same/Zero is actually supported. If a device rejects
1136 * a non-read/write command (discard, write same,etc.) the
1137 * low-level device driver will set the relevant queue limit to
1138 * 0 to prevent blk-lib from issuing more of the offending
1139 * operations. Commands queued prior to the queue limit being
1140 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
1141 * errors being propagated to upper layers.
1142 */
1143 if (max_sectors == 0)
1144 return BLK_STS_NOTSUPP;
1145
John Pittman61939b12019-05-23 17:49:39 -04001146 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
Ritika Srivastava8327cce52020-09-01 13:17:31 -07001147 __func__, blk_rq_sectors(rq), max_sectors);
Ritika Srivastava143d2602020-09-01 13:17:30 -07001148 return BLK_STS_IOERR;
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001149 }
1150
1151 /*
Christoph Hellwig9bb33f22021-03-31 09:30:00 +02001152 * The queue settings related to segment counting may differ from the
1153 * original queue.
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001154 */
Christoph Hellwige9cd19c2019-06-06 12:29:02 +02001155 rq->nr_phys_segments = blk_recalc_rq_segments(rq);
Martin K. Petersen8a783622010-02-26 00:20:39 -05001156 if (rq->nr_phys_segments > queue_max_segments(q)) {
John Pittman61939b12019-05-23 17:49:39 -04001157 printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
1158 __func__, rq->nr_phys_segments, queue_max_segments(q));
Ritika Srivastava143d2602020-09-01 13:17:30 -07001159 return BLK_STS_IOERR;
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001160 }
1161
Ritika Srivastava143d2602020-09-01 13:17:30 -07001162 return BLK_STS_OK;
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001163}
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001164
1165/**
1166 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1167 * @q: the queue to submit the request
1168 * @rq: the request being queued
1169 */
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001170blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001171{
Ritika Srivastava8327cce52020-09-01 13:17:31 -07001172 blk_status_t ret;
1173
1174 ret = blk_cloned_rq_check_limits(q, rq);
1175 if (ret != BLK_STS_OK)
1176 return ret;
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001177
Akinobu Mitab2c9cd32011-07-26 16:09:03 -07001178 if (rq->rq_disk &&
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001179 should_fail_request(rq->rq_disk->part0, blk_rq_bytes(rq)))
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001180 return BLK_STS_IOERR;
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001181
Satya Tangiralaa892c8d2020-05-14 00:37:18 +00001182 if (blk_crypto_insert_cloned_request(rq))
1183 return BLK_STS_IOERR;
1184
Jens Axboea1ce35f2018-10-29 10:23:51 -06001185 if (blk_queue_io_stat(q))
Konstantin Khlebnikovb5af37a2020-05-27 07:24:16 +02001186 blk_account_io_start(rq);
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001187
1188 /*
Jens Axboea1ce35f2018-10-29 10:23:51 -06001189 * Since we have a scheduler attached on the top device,
1190 * bypass a potential scheduler on the bottom device for
1191 * insert.
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001192 */
Bart Van Asschefd9c40f2019-04-04 10:08:43 -07001193 return blk_mq_request_issue_directly(rq, true);
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001194}
1195EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1196
Tejun Heo80a761f2009-07-03 17:48:17 +09001197/**
1198 * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1199 * @rq: request to examine
1200 *
1201 * Description:
1202 * A request could be merge of IOs which require different failure
1203 * handling. This function determines the number of bytes which
1204 * can be failed from the beginning of the request without
1205 * crossing into area which need to be retried further.
1206 *
1207 * Return:
1208 * The number of bytes to fail.
Tejun Heo80a761f2009-07-03 17:48:17 +09001209 */
1210unsigned int blk_rq_err_bytes(const struct request *rq)
1211{
1212 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1213 unsigned int bytes = 0;
1214 struct bio *bio;
1215
Christoph Hellwige8064022016-10-20 15:12:13 +02001216 if (!(rq->rq_flags & RQF_MIXED_MERGE))
Tejun Heo80a761f2009-07-03 17:48:17 +09001217 return blk_rq_bytes(rq);
1218
1219 /*
1220 * Currently the only 'mixing' which can happen is between
1221 * different fastfail types. We can safely fail portions
1222 * which have all the failfast bits that the first one has -
1223 * the ones which are at least as eager to fail as the first
1224 * one.
1225 */
1226 for (bio = rq->bio; bio; bio = bio->bi_next) {
Jens Axboe1eff9d32016-08-05 15:35:16 -06001227 if ((bio->bi_opf & ff) != ff)
Tejun Heo80a761f2009-07-03 17:48:17 +09001228 break;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001229 bytes += bio->bi_iter.bi_size;
Tejun Heo80a761f2009-07-03 17:48:17 +09001230 }
1231
1232 /* this could lead to infinite loop */
1233 BUG_ON(blk_rq_bytes(rq) && !bytes);
1234 return bytes;
1235}
1236EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
1237
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001238static void update_io_ticks(struct block_device *part, unsigned long now,
1239 bool end)
Christoph Hellwig9123bf62020-05-27 07:24:13 +02001240{
1241 unsigned long stamp;
1242again:
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001243 stamp = READ_ONCE(part->bd_stamp);
Chunguang Xud80c2282021-07-06 05:47:26 +08001244 if (unlikely(time_after(now, stamp))) {
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001245 if (likely(cmpxchg(&part->bd_stamp, stamp, now) == stamp))
Christoph Hellwig9123bf62020-05-27 07:24:13 +02001246 __part_stat_add(part, io_ticks, end ? now - stamp : 1);
1247 }
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001248 if (part->bd_partno) {
1249 part = bdev_whole(part);
Christoph Hellwig9123bf62020-05-27 07:24:13 +02001250 goto again;
1251 }
1252}
1253
Christoph Hellwigf1394b792020-05-13 12:49:32 +02001254static void blk_account_io_completion(struct request *req, unsigned int bytes)
Jens Axboebc58ba92009-01-23 10:54:44 +01001255{
Logan Gunthorpeecb61862019-12-10 11:47:04 -07001256 if (req->part && blk_do_io_stat(req)) {
Michael Callahanddcf35d2018-07-18 04:47:39 -07001257 const int sgrp = op_stat_group(req_op(req));
Jens Axboebc58ba92009-01-23 10:54:44 +01001258
Mike Snitzer112f1582018-12-06 11:41:18 -05001259 part_stat_lock();
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001260 part_stat_add(req->part, sectors[sgrp], bytes >> 9);
Jens Axboebc58ba92009-01-23 10:54:44 +01001261 part_stat_unlock();
1262 }
1263}
1264
Omar Sandoval522a7772018-05-09 02:08:53 -07001265void blk_account_io_done(struct request *req, u64 now)
Jens Axboebc58ba92009-01-23 10:54:44 +01001266{
Jens Axboebc58ba92009-01-23 10:54:44 +01001267 /*
Tejun Heodd4c1332010-09-03 11:56:16 +02001268 * Account IO completion. flush_rq isn't accounted as a
1269 * normal IO on queueing nor completion. Accounting the
1270 * containing request is enough.
Jens Axboebc58ba92009-01-23 10:54:44 +01001271 */
Logan Gunthorpeecb61862019-12-10 11:47:04 -07001272 if (req->part && blk_do_io_stat(req) &&
1273 !(req->rq_flags & RQF_FLUSH_SEQ)) {
Michael Callahanddcf35d2018-07-18 04:47:39 -07001274 const int sgrp = op_stat_group(req_op(req));
Jens Axboebc58ba92009-01-23 10:54:44 +01001275
Mike Snitzer112f1582018-12-06 11:41:18 -05001276 part_stat_lock();
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001277 update_io_ticks(req->part, jiffies, true);
1278 part_stat_inc(req->part, ios[sgrp]);
1279 part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
Christoph Hellwig524f9ff2020-05-27 07:24:19 +02001280 part_stat_unlock();
Jens Axboebc58ba92009-01-23 10:54:44 +01001281 }
1282}
1283
Konstantin Khlebnikovb5af37a2020-05-27 07:24:16 +02001284void blk_account_io_start(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +01001285{
Jens Axboe320ae512013-10-24 09:20:05 +01001286 if (!blk_do_io_stat(rq))
1287 return;
1288
Christoph Hellwig0b6e5222021-01-24 11:02:38 +01001289 /* passthrough requests can hold bios that do not have ->bi_bdev set */
1290 if (rq->bio && rq->bio->bi_bdev)
1291 rq->part = rq->bio->bi_bdev;
1292 else
1293 rq->part = rq->rq_disk->part0;
Christoph Hellwig524f9ff2020-05-27 07:24:19 +02001294
Mike Snitzer112f1582018-12-06 11:41:18 -05001295 part_stat_lock();
Christoph Hellwig76268f32020-05-13 12:49:34 +02001296 update_io_ticks(rq->part, jiffies, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001297 part_stat_unlock();
1298}
1299
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001300static unsigned long __part_start_io_acct(struct block_device *part,
Song Liu7b264102020-08-31 15:27:23 -07001301 unsigned int sectors, unsigned int op)
Christoph Hellwig956d5102020-05-27 07:24:04 +02001302{
Christoph Hellwig956d5102020-05-27 07:24:04 +02001303 const int sgrp = op_stat_group(op);
1304 unsigned long now = READ_ONCE(jiffies);
1305
1306 part_stat_lock();
1307 update_io_ticks(part, now, false);
1308 part_stat_inc(part, ios[sgrp]);
1309 part_stat_add(part, sectors[sgrp], sectors);
1310 part_stat_local_inc(part, in_flight[op_is_write(op)]);
1311 part_stat_unlock();
1312
1313 return now;
1314}
Song Liu7b264102020-08-31 15:27:23 -07001315
Christoph Hellwig99dfc432021-01-24 11:02:37 +01001316/**
1317 * bio_start_io_acct - start I/O accounting for bio based drivers
1318 * @bio: bio to start account for
1319 *
1320 * Returns the start time that should be passed back to bio_end_io_acct().
1321 */
1322unsigned long bio_start_io_acct(struct bio *bio)
Song Liu7b264102020-08-31 15:27:23 -07001323{
Christoph Hellwig99dfc432021-01-24 11:02:37 +01001324 return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio));
Song Liu7b264102020-08-31 15:27:23 -07001325}
Christoph Hellwig99dfc432021-01-24 11:02:37 +01001326EXPORT_SYMBOL_GPL(bio_start_io_acct);
Song Liu7b264102020-08-31 15:27:23 -07001327
1328unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1329 unsigned int op)
1330{
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001331 return __part_start_io_acct(disk->part0, sectors, op);
Song Liu7b264102020-08-31 15:27:23 -07001332}
Christoph Hellwig956d5102020-05-27 07:24:04 +02001333EXPORT_SYMBOL(disk_start_io_acct);
1334
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001335static void __part_end_io_acct(struct block_device *part, unsigned int op,
Song Liu7b264102020-08-31 15:27:23 -07001336 unsigned long start_time)
Christoph Hellwig956d5102020-05-27 07:24:04 +02001337{
Christoph Hellwig956d5102020-05-27 07:24:04 +02001338 const int sgrp = op_stat_group(op);
1339 unsigned long now = READ_ONCE(jiffies);
1340 unsigned long duration = now - start_time;
1341
1342 part_stat_lock();
1343 update_io_ticks(part, now, true);
1344 part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration));
1345 part_stat_local_dec(part, in_flight[op_is_write(op)]);
1346 part_stat_unlock();
1347}
Song Liu7b264102020-08-31 15:27:23 -07001348
Christoph Hellwig99dfc432021-01-24 11:02:37 +01001349void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1350 struct block_device *orig_bdev)
Song Liu7b264102020-08-31 15:27:23 -07001351{
Christoph Hellwig99dfc432021-01-24 11:02:37 +01001352 __part_end_io_acct(orig_bdev, bio_op(bio), start_time);
Song Liu7b264102020-08-31 15:27:23 -07001353}
Christoph Hellwig99dfc432021-01-24 11:02:37 +01001354EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped);
Song Liu7b264102020-08-31 15:27:23 -07001355
1356void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1357 unsigned long start_time)
1358{
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001359 __part_end_io_acct(disk->part0, op, start_time);
Song Liu7b264102020-08-31 15:27:23 -07001360}
Christoph Hellwig956d5102020-05-27 07:24:04 +02001361EXPORT_SYMBOL(disk_end_io_acct);
1362
Christoph Hellwigef71de82017-11-02 21:29:51 +03001363/*
1364 * Steal bios from a request and add them to a bio list.
1365 * The request must not have been partially completed before.
1366 */
1367void blk_steal_bios(struct bio_list *list, struct request *rq)
1368{
1369 if (rq->bio) {
1370 if (list->tail)
1371 list->tail->bi_next = rq->bio;
1372 else
1373 list->head = rq->bio;
1374 list->tail = rq->biotail;
1375
1376 rq->bio = NULL;
1377 rq->biotail = NULL;
1378 }
1379
1380 rq->__data_len = 0;
1381}
1382EXPORT_SYMBOL_GPL(blk_steal_bios);
1383
Tejun Heo9934c8c2009-05-08 11:54:16 +09001384/**
Bart Van Assche7cc26232021-05-19 10:52:26 -07001385 * blk_update_request - Complete multiple bytes without completing the request
Randy Dunlap8ebf9752009-06-11 20:00:41 -07001386 * @req: the request being processed
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001387 * @error: block status code
Bart Van Assche7cc26232021-05-19 10:52:26 -07001388 * @nr_bytes: number of bytes to complete for @req
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001389 *
1390 * Description:
Randy Dunlap8ebf9752009-06-11 20:00:41 -07001391 * Ends I/O on a number of bytes attached to @req, but doesn't complete
1392 * the request structure even if @req doesn't have leftover.
1393 * If @req has leftover, sets it up for the next range of segments.
Tejun Heo2e60e022009-04-23 11:05:18 +09001394 *
Tejun Heo2e60e022009-04-23 11:05:18 +09001395 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
1396 * %false return from this function.
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001397 *
Bart Van Assche1954e9a2018-06-27 13:09:05 -07001398 * Note:
Bart Van Assche7cc26232021-05-19 10:52:26 -07001399 * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
1400 * except in the consistency check at the end of this function.
Bart Van Assche1954e9a2018-06-27 13:09:05 -07001401 *
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001402 * Return:
Tejun Heo2e60e022009-04-23 11:05:18 +09001403 * %false - this request doesn't have any more data
1404 * %true - this request has more data
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001405 **/
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001406bool blk_update_request(struct request *req, blk_status_t error,
1407 unsigned int nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408{
Kent Overstreetf79ea412012-09-20 16:38:30 -07001409 int total_bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001411 trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
Hannes Reinecke4a0efdc2014-10-01 14:32:31 +02001412
Tejun Heo2e60e022009-04-23 11:05:18 +09001413 if (!req->bio)
1414 return false;
1415
Max Gurtovoy54d4e6a2019-09-16 18:44:29 +03001416#ifdef CONFIG_BLK_DEV_INTEGRITY
1417 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
1418 error == BLK_STS_OK)
1419 req->q->integrity.profile->complete_fn(req, nr_bytes);
1420#endif
1421
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001422 if (unlikely(error && !blk_rq_is_passthrough(req) &&
1423 !(req->rq_flags & RQF_QUIET)))
Christoph Hellwig178cc592019-06-20 10:59:15 -07001424 print_req_error(req, error, __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425
Jens Axboebc58ba92009-01-23 10:54:44 +01001426 blk_account_io_completion(req, nr_bytes);
Jens Axboed72d9042005-11-01 08:35:42 +01001427
Kent Overstreetf79ea412012-09-20 16:38:30 -07001428 total_bytes = 0;
1429 while (req->bio) {
1430 struct bio *bio = req->bio;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001431 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432
Bart Van Assche9c24c102018-06-19 10:26:40 -07001433 if (bio_bytes == bio->bi_iter.bi_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434 req->bio = bio->bi_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435
NeilBrownfbbaf702017-04-07 09:40:52 -06001436 /* Completion has already been traced */
1437 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
Kent Overstreetf79ea412012-09-20 16:38:30 -07001438 req_bio_endio(req, bio, bio_bytes, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439
Kent Overstreetf79ea412012-09-20 16:38:30 -07001440 total_bytes += bio_bytes;
1441 nr_bytes -= bio_bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442
Kent Overstreetf79ea412012-09-20 16:38:30 -07001443 if (!nr_bytes)
1444 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 }
1446
1447 /*
1448 * completely done
1449 */
Tejun Heo2e60e022009-04-23 11:05:18 +09001450 if (!req->bio) {
1451 /*
1452 * Reset counters so that the request stacking driver
1453 * can find how many bytes remain in the request
1454 * later.
1455 */
Tejun Heoa2dec7b2009-05-07 22:24:44 +09001456 req->__data_len = 0;
Tejun Heo2e60e022009-04-23 11:05:18 +09001457 return false;
1458 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459
Tejun Heoa2dec7b2009-05-07 22:24:44 +09001460 req->__data_len -= total_bytes;
Tejun Heo2e46e8b2009-05-07 22:24:41 +09001461
1462 /* update sector only for requests with clear definition of sector */
Christoph Hellwig57292b52017-01-31 16:57:29 +01001463 if (!blk_rq_is_passthrough(req))
Tejun Heoa2dec7b2009-05-07 22:24:44 +09001464 req->__sector += total_bytes >> 9;
Tejun Heo2e46e8b2009-05-07 22:24:41 +09001465
Tejun Heo80a761f2009-07-03 17:48:17 +09001466 /* mixed attributes always follow the first bio */
Christoph Hellwige8064022016-10-20 15:12:13 +02001467 if (req->rq_flags & RQF_MIXED_MERGE) {
Tejun Heo80a761f2009-07-03 17:48:17 +09001468 req->cmd_flags &= ~REQ_FAILFAST_MASK;
Jens Axboe1eff9d32016-08-05 15:35:16 -06001469 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
Tejun Heo80a761f2009-07-03 17:48:17 +09001470 }
1471
Christoph Hellwiged6565e2017-05-11 12:34:38 +02001472 if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
1473 /*
1474 * If total number of sectors is less than the first segment
1475 * size, something has gone terribly wrong.
1476 */
1477 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
1478 blk_dump_rq_flags(req, "request botched");
1479 req->__data_len = blk_rq_cur_bytes(req);
1480 }
Tejun Heo2e46e8b2009-05-07 22:24:41 +09001481
Christoph Hellwiged6565e2017-05-11 12:34:38 +02001482 /* recalculate the number of segments */
Christoph Hellwige9cd19c2019-06-06 12:29:02 +02001483 req->nr_phys_segments = blk_recalc_rq_segments(req);
Christoph Hellwiged6565e2017-05-11 12:34:38 +02001484 }
Tejun Heo2e46e8b2009-05-07 22:24:41 +09001485
Tejun Heo2e60e022009-04-23 11:05:18 +09001486 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487}
Tejun Heo2e60e022009-04-23 11:05:18 +09001488EXPORT_SYMBOL_GPL(blk_update_request);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489
Ilya Loginov2d4dc892009-11-26 09:16:19 +01001490#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1491/**
1492 * rq_flush_dcache_pages - Helper function to flush all pages in a request
1493 * @rq: the request to be flushed
1494 *
1495 * Description:
1496 * Flush all pages in @rq.
1497 */
1498void rq_flush_dcache_pages(struct request *rq)
1499{
1500 struct req_iterator iter;
Kent Overstreet79886132013-11-23 17:19:00 -08001501 struct bio_vec bvec;
Ilya Loginov2d4dc892009-11-26 09:16:19 +01001502
1503 rq_for_each_segment(bvec, rq, iter)
Kent Overstreet79886132013-11-23 17:19:00 -08001504 flush_dcache_page(bvec.bv_page);
Ilya Loginov2d4dc892009-11-26 09:16:19 +01001505}
1506EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
1507#endif
1508
Kiyoshi Uedaef9e3fa2008-10-01 16:12:15 +02001509/**
1510 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
1511 * @q : the queue of the device being checked
1512 *
1513 * Description:
1514 * Check if underlying low-level drivers of a device are busy.
1515 * If the drivers want to export their busy state, they must set own
1516 * exporting function using blk_queue_lld_busy() first.
1517 *
1518 * Basically, this function is used only by request stacking drivers
1519 * to stop dispatching requests to underlying devices when underlying
1520 * devices are busy. This behavior helps more I/O merging on the queue
1521 * of the request stacking driver and prevents I/O throughput regression
1522 * on burst I/O load.
1523 *
1524 * Return:
1525 * 0 - Not busy (The request stacking driver should dispatch request)
1526 * 1 - Busy (The request stacking driver should stop dispatching request)
1527 */
1528int blk_lld_busy(struct request_queue *q)
1529{
Jens Axboe344e9ff2018-11-15 12:22:51 -07001530 if (queue_is_mq(q) && q->mq_ops->busy)
Jens Axboe9ba20522018-10-29 10:15:10 -06001531 return q->mq_ops->busy(q);
Kiyoshi Uedaef9e3fa2008-10-01 16:12:15 +02001532
1533 return 0;
1534}
1535EXPORT_SYMBOL_GPL(blk_lld_busy);
1536
Mike Snitzer78d8e582015-06-26 10:01:13 -04001537/**
1538 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
1539 * @rq: the clone request to be cleaned up
1540 *
1541 * Description:
1542 * Free all bios in @rq for a cloned request.
1543 */
1544void blk_rq_unprep_clone(struct request *rq)
1545{
1546 struct bio *bio;
1547
1548 while ((bio = rq->bio) != NULL) {
1549 rq->bio = bio->bi_next;
1550
1551 bio_put(bio);
1552 }
1553}
1554EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
1555
Mike Snitzer78d8e582015-06-26 10:01:13 -04001556/**
1557 * blk_rq_prep_clone - Helper function to setup clone request
1558 * @rq: the request to be setup
1559 * @rq_src: original request to be cloned
1560 * @bs: bio_set that bios for clone are allocated from
1561 * @gfp_mask: memory allocation mask for bio
1562 * @bio_ctr: setup function to be called for each clone bio.
1563 * Returns %0 for success, non %0 for failure.
1564 * @data: private data to be passed to @bio_ctr
1565 *
1566 * Description:
1567 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
Mike Snitzer78d8e582015-06-26 10:01:13 -04001568 * Also, pages which the original bios are pointing to are not copied
1569 * and the cloned bios just point same pages.
1570 * So cloned bios must be completed before original bios, which means
1571 * the caller must complete @rq before @rq_src.
1572 */
1573int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
1574 struct bio_set *bs, gfp_t gfp_mask,
1575 int (*bio_ctr)(struct bio *, struct bio *, void *),
1576 void *data)
1577{
1578 struct bio *bio, *bio_src;
1579
1580 if (!bs)
Kent Overstreetf4f81542018-05-08 21:33:52 -04001581 bs = &fs_bio_set;
Mike Snitzer78d8e582015-06-26 10:01:13 -04001582
1583 __rq_for_each_bio(bio_src, rq_src) {
1584 bio = bio_clone_fast(bio_src, gfp_mask, bs);
1585 if (!bio)
1586 goto free_and_out;
1587
1588 if (bio_ctr && bio_ctr(bio, bio_src, data))
1589 goto free_and_out;
1590
1591 if (rq->bio) {
1592 rq->biotail->bi_next = bio;
1593 rq->biotail = bio;
Eric Biggers93f221a2020-09-15 20:53:14 -07001594 } else {
Mike Snitzer78d8e582015-06-26 10:01:13 -04001595 rq->bio = rq->biotail = bio;
Eric Biggers93f221a2020-09-15 20:53:14 -07001596 }
1597 bio = NULL;
Mike Snitzer78d8e582015-06-26 10:01:13 -04001598 }
1599
Guoqing Jiang361301a2020-03-09 22:41:36 +01001600 /* Copy attributes of the original request to the clone request. */
1601 rq->__sector = blk_rq_pos(rq_src);
1602 rq->__data_len = blk_rq_bytes(rq_src);
1603 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
1604 rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
1605 rq->special_vec = rq_src->special_vec;
1606 }
1607 rq->nr_phys_segments = rq_src->nr_phys_segments;
1608 rq->ioprio = rq_src->ioprio;
Mike Snitzer78d8e582015-06-26 10:01:13 -04001609
Eric Biggers93f221a2020-09-15 20:53:14 -07001610 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
1611 goto free_and_out;
Mike Snitzer78d8e582015-06-26 10:01:13 -04001612
1613 return 0;
1614
1615free_and_out:
1616 if (bio)
1617 bio_put(bio);
1618 blk_rq_unprep_clone(rq);
1619
1620 return -ENOMEM;
Kiyoshi Uedab0fd2712009-06-11 13:10:16 +02001621}
1622EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
1623
Jens Axboe59c3d452014-04-08 09:15:35 -06001624int kblockd_schedule_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625{
1626 return queue_work(kblockd_workqueue, work);
1627}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628EXPORT_SYMBOL(kblockd_schedule_work);
1629
Jens Axboe818cd1c2017-04-10 09:54:55 -06001630int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
1631 unsigned long delay)
1632{
1633 return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
1634}
1635EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
1636
Suresh Jayaraman75df7132011-09-21 10:00:16 +02001637/**
1638 * blk_start_plug - initialize blk_plug and track it inside the task_struct
1639 * @plug: The &struct blk_plug that needs to be initialized
1640 *
1641 * Description:
Jeff Moyer40405852019-01-08 16:57:34 -05001642 * blk_start_plug() indicates to the block layer an intent by the caller
1643 * to submit multiple I/O requests in a batch. The block layer may use
1644 * this hint to defer submitting I/Os from the caller until blk_finish_plug()
1645 * is called. However, the block layer may choose to submit requests
1646 * before a call to blk_finish_plug() if the number of queued I/Os
1647 * exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than
1648 * %BLK_PLUG_FLUSH_SIZE. The queued I/Os may also be submitted early if
1649 * the task schedules (see below).
1650 *
Suresh Jayaraman75df7132011-09-21 10:00:16 +02001651 * Tracking blk_plug inside the task_struct will help with auto-flushing the
1652 * pending I/O should the task end up blocking between blk_start_plug() and
1653 * blk_finish_plug(). This is important from a performance perspective, but
1654 * also ensures that we don't deadlock. For instance, if the task is blocking
1655 * for a memory allocation, memory reclaim could end up wanting to free a
1656 * page belonging to that request that is currently residing in our private
1657 * plug. By flushing the pending I/O when the process goes to sleep, we avoid
1658 * this kind of deadlock.
1659 */
Jens Axboe73c10102011-03-08 13:19:51 +01001660void blk_start_plug(struct blk_plug *plug)
1661{
1662 struct task_struct *tsk = current;
1663
Shaohua Lidd6cf3e2015-05-08 10:51:28 -07001664 /*
1665 * If this is a nested plug, don't actually assign it.
1666 */
1667 if (tsk->plug)
1668 return;
1669
Jens Axboe320ae512013-10-24 09:20:05 +01001670 INIT_LIST_HEAD(&plug->mq_list);
NeilBrown048c9372011-04-18 09:52:22 +02001671 INIT_LIST_HEAD(&plug->cb_list);
Jens Axboe5f0ed772018-11-23 22:04:33 -07001672 plug->rq_count = 0;
Jens Axboece5b0092018-11-27 17:13:56 -07001673 plug->multiple_queues = false;
Jens Axboe5a473e82020-06-04 11:23:39 -06001674 plug->nowait = false;
Jens Axboe5f0ed772018-11-23 22:04:33 -07001675
Jens Axboe73c10102011-03-08 13:19:51 +01001676 /*
Shaohua Lidd6cf3e2015-05-08 10:51:28 -07001677 * Store ordering should not be needed here, since a potential
1678 * preempt will imply a full memory barrier
Jens Axboe73c10102011-03-08 13:19:51 +01001679 */
Shaohua Lidd6cf3e2015-05-08 10:51:28 -07001680 tsk->plug = plug;
Jens Axboe73c10102011-03-08 13:19:51 +01001681}
1682EXPORT_SYMBOL(blk_start_plug);
1683
NeilBrown74018dc2012-07-31 09:08:15 +02001684static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
NeilBrown048c9372011-04-18 09:52:22 +02001685{
1686 LIST_HEAD(callbacks);
1687
Shaohua Li2a7d5552012-07-31 09:08:15 +02001688 while (!list_empty(&plug->cb_list)) {
1689 list_splice_init(&plug->cb_list, &callbacks);
NeilBrown048c9372011-04-18 09:52:22 +02001690
Shaohua Li2a7d5552012-07-31 09:08:15 +02001691 while (!list_empty(&callbacks)) {
1692 struct blk_plug_cb *cb = list_first_entry(&callbacks,
NeilBrown048c9372011-04-18 09:52:22 +02001693 struct blk_plug_cb,
1694 list);
Shaohua Li2a7d5552012-07-31 09:08:15 +02001695 list_del(&cb->list);
NeilBrown74018dc2012-07-31 09:08:15 +02001696 cb->callback(cb, from_schedule);
Shaohua Li2a7d5552012-07-31 09:08:15 +02001697 }
NeilBrown048c9372011-04-18 09:52:22 +02001698 }
1699}
1700
NeilBrown9cbb1752012-07-31 09:08:14 +02001701struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
1702 int size)
1703{
1704 struct blk_plug *plug = current->plug;
1705 struct blk_plug_cb *cb;
1706
1707 if (!plug)
1708 return NULL;
1709
1710 list_for_each_entry(cb, &plug->cb_list, list)
1711 if (cb->callback == unplug && cb->data == data)
1712 return cb;
1713
1714 /* Not currently on the callback list */
1715 BUG_ON(size < sizeof(*cb));
1716 cb = kzalloc(size, GFP_ATOMIC);
1717 if (cb) {
1718 cb->data = data;
1719 cb->callback = unplug;
1720 list_add(&cb->list, &plug->cb_list);
1721 }
1722 return cb;
1723}
1724EXPORT_SYMBOL(blk_check_plugged);
1725
Jens Axboe49cac012011-04-16 13:51:05 +02001726void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
Jens Axboe73c10102011-03-08 13:19:51 +01001727{
NeilBrown74018dc2012-07-31 09:08:15 +02001728 flush_plug_callbacks(plug, from_schedule);
Jens Axboe320ae512013-10-24 09:20:05 +01001729
1730 if (!list_empty(&plug->mq_list))
1731 blk_mq_flush_plug_list(plug, from_schedule);
Jens Axboe73c10102011-03-08 13:19:51 +01001732}
Jens Axboe73c10102011-03-08 13:19:51 +01001733
Jeff Moyer40405852019-01-08 16:57:34 -05001734/**
1735 * blk_finish_plug - mark the end of a batch of submitted I/O
1736 * @plug: The &struct blk_plug passed to blk_start_plug()
1737 *
1738 * Description:
1739 * Indicate that a batch of I/O submissions is complete. This function
1740 * must be paired with an initial call to blk_start_plug(). The intent
1741 * is to allow the block layer to optimize I/O submission. See the
1742 * documentation for blk_start_plug() for more information.
1743 */
Jens Axboe73c10102011-03-08 13:19:51 +01001744void blk_finish_plug(struct blk_plug *plug)
1745{
Shaohua Lidd6cf3e2015-05-08 10:51:28 -07001746 if (plug != current->plug)
1747 return;
Jens Axboef6603782011-04-15 15:49:07 +02001748 blk_flush_plug_list(plug, false);
Christoph Hellwig88b996c2011-04-15 15:20:10 +02001749
Shaohua Lidd6cf3e2015-05-08 10:51:28 -07001750 current->plug = NULL;
Jens Axboe73c10102011-03-08 13:19:51 +01001751}
1752EXPORT_SYMBOL(blk_finish_plug);
1753
Ming Lei71ac8602020-05-14 16:45:09 +08001754void blk_io_schedule(void)
1755{
1756 /* Prevent hang_check timer from firing at us during very long I/O */
1757 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;
1758
1759 if (timeout)
1760 io_schedule_timeout(timeout);
1761 else
1762 io_schedule();
1763}
1764EXPORT_SYMBOL_GPL(blk_io_schedule);
1765
Linus Torvalds1da177e2005-04-16 15:20:36 -07001766int __init blk_dev_init(void)
1767{
Christoph Hellwigef295ec2016-10-28 08:48:16 -06001768 BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
1769 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
Pankaj Bharadiyac5936422019-12-09 10:31:43 -08001770 sizeof_field(struct request, cmd_flags));
Christoph Hellwigef295ec2016-10-28 08:48:16 -06001771 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
Pankaj Bharadiyac5936422019-12-09 10:31:43 -08001772 sizeof_field(struct bio, bi_opf));
Nikanth Karthikesan9eb55b02009-04-27 14:53:54 +02001773
Tejun Heo89b90be2011-01-03 15:01:47 +01001774 /* used for unplugging and affects IO latency/throughput - HIGHPRI */
1775 kblockd_workqueue = alloc_workqueue("kblockd",
Matias Bjørling28747fc2014-06-11 23:43:54 +02001776 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001777 if (!kblockd_workqueue)
1778 panic("Failed to create kblockd\n");
1779
Ilya Dryomovc2789bd2015-11-20 22:16:46 +01001780 blk_requestq_cachep = kmem_cache_create("request_queue",
Jens Axboe165125e2007-07-24 09:28:11 +02001781 sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782
Omar Sandoval18fbda92017-01-31 14:53:20 -08001783 blk_debugfs_root = debugfs_create_dir("block", NULL);
Omar Sandoval18fbda92017-01-31 14:53:20 -08001784
Linus Torvalds1da177e2005-04-16 15:20:36 -07001785 return 0;
1786}